You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2013/10/04 23:30:46 UTC
svn commit: r1529308 [5/10] - in /hive/branches/maven: ./
ant/src/org/apache/hadoop/hive/ant/ beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/ bin/ common/
common/src/java/org/apache/hadoop/hive/conf/ common/...
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapper.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapper.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapper.java Fri Oct 4 21:30:38 2013
@@ -22,9 +22,9 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
public abstract class KeyWrapper {
- abstract void getNewKey(Object row, ObjectInspector rowInspector) throws HiveException;
- abstract void setHashKey();
- abstract KeyWrapper copyKey();
- abstract void copyKey(KeyWrapper oldWrapper);
- abstract Object[] getKeyArray();
+ public abstract void getNewKey(Object row, ObjectInspector rowInspector) throws HiveException;
+ public abstract void setHashKey();
+ public abstract KeyWrapper copyKey();
+ public abstract void copyKey(KeyWrapper oldWrapper);
+ public abstract Object[] getKeyArray();
}
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Fri Oct 4 21:30:38 2013
@@ -101,7 +101,7 @@ public abstract class Operator<T extends
protected transient State state = State.UNINIT;
- static transient boolean fatalError = false; // fatalError is shared acorss
+ protected static transient boolean fatalError = false; // fatalError is shared acorss
// all operators
static {
@@ -1438,6 +1438,60 @@ public abstract class Operator<T extends
return ret;
}
+ /**
+ * Clones only the operator. The children and parent are set
+ * to null.
+ * @return Cloned operator
+ * @throws CloneNotSupportedException
+ */
+ public Operator<? extends OperatorDesc> cloneOp() throws CloneNotSupportedException {
+ T descClone = (T) conf.clone();
+ Operator<? extends OperatorDesc> ret =
+ (Operator<? extends OperatorDesc>) OperatorFactory.getAndMakeChild(
+ descClone, getSchema());
+ return ret;
+ }
+
+ /**
+ * Recursively clones all the children of the tree,
+ * Fixes the pointers to children, parents and the pointers to itself coming from the children.
+ * It does not fix the pointers to itself coming from parents, parents continue to point to
+ * the original child.
+ * @return Cloned operator
+ * @throws CloneNotSupportedException
+ */
+ public Operator<? extends OperatorDesc> cloneRecursiveChildren()
+ throws CloneNotSupportedException {
+ Operator<? extends OperatorDesc> newOp = this.cloneOp();
+ newOp.setParentOperators(this.parentOperators);
+ // Fix parent in all children
+ if (this.getChildOperators() == null) {
+ newOp.setChildOperators(null);
+ return newOp;
+ }
+ List<Operator<? extends OperatorDesc>> newChildren =
+ new ArrayList<Operator<? extends OperatorDesc>>();
+
+ for (Operator<? extends OperatorDesc> childOp : this.getChildOperators()) {
+ List<Operator<? extends OperatorDesc>> parentList =
+ new ArrayList<Operator<? extends OperatorDesc>>();
+ for (Operator<? extends OperatorDesc> parent : childOp.getParentOperators()) {
+ if (parent.equals(this)) {
+ parentList.add(newOp);
+ } else {
+ parentList.add(parent);
+ }
+ }
+ // Recursively clone the children
+ Operator<? extends OperatorDesc> clonedChildOp = childOp.cloneRecursiveChildren();
+ clonedChildOp.setParentOperators(parentList);
+ }
+
+ newOp.setChildOperators(newChildren);
+ return newOp;
+ }
+
+
/*
* True only for operators which produce atmost 1 output row per input
* row to it. This will allow the output column names to be directly
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java Fri Oct 4 21:30:38 2013
@@ -22,12 +22,17 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.ql.exec.vector.VectorFileSinkOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
import org.apache.hadoop.hive.ql.plan.CollectDesc;
-import org.apache.hadoop.hive.ql.plan.MuxDesc;
import org.apache.hadoop.hive.ql.plan.DemuxDesc;
import org.apache.hadoop.hive.ql.plan.DummyStoreDesc;
-import org.apache.hadoop.hive.ql.plan.ExtractDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExtractDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
import org.apache.hadoop.hive.ql.plan.ForwardDesc;
@@ -39,6 +44,7 @@ import org.apache.hadoop.hive.ql.plan.La
import org.apache.hadoop.hive.ql.plan.LateralViewJoinDesc;
import org.apache.hadoop.hive.ql.plan.LimitDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.MuxDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PTFDesc;
import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
@@ -106,6 +112,38 @@ public final class OperatorFactory {
MuxOperator.class));
}
+ public static ArrayList<OpTuple> vectorOpvec;
+ static {
+ vectorOpvec = new ArrayList<OpTuple>();
+ vectorOpvec.add(new OpTuple<SelectDesc>(SelectDesc.class, VectorSelectOperator.class));
+ vectorOpvec.add(new OpTuple<GroupByDesc>(GroupByDesc.class, VectorGroupByOperator.class));
+ vectorOpvec.add(new OpTuple<ReduceSinkDesc>(ReduceSinkDesc.class,
+ VectorReduceSinkOperator.class));
+ vectorOpvec.add(new OpTuple<FileSinkDesc>(FileSinkDesc.class, VectorFileSinkOperator.class));
+ vectorOpvec.add(new OpTuple<FilterDesc>(FilterDesc.class, VectorFilterOperator.class));
+ }
+
+ public static <T extends OperatorDesc> Operator<T> getVectorOperator(T conf,
+ VectorizationContext vContext) {
+ Class<T> descClass = (Class<T>) conf.getClass();
+ for (OpTuple o : vectorOpvec) {
+ if (o.descClass == descClass) {
+ try {
+ Operator<T> op = (Operator<T>) o.opClass.getDeclaredConstructor(
+ VectorizationContext.class, OperatorDesc.class).newInstance(
+ vContext, conf);
+ op.initializeCounters();
+ return op;
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ throw new RuntimeException("No vector operator for descriptor class "
+ + descClass.getName());
+ }
+
public static <T extends OperatorDesc> Operator<T> get(Class<T> opClass) {
for (OpTuple o : opvec) {
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java Fri Oct 4 21:30:38 2013
@@ -19,9 +19,9 @@
package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
-import java.util.ArrayList;
+import java.util.ArrayDeque;
+import java.util.Deque;
import java.util.List;
-import java.util.Stack;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -30,12 +30,12 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFInputDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
import org.apache.hadoop.hive.ql.plan.PTFDeserializer;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PartitionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag;
import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator;
import org.apache.hadoop.hive.serde2.SerDe;
@@ -147,7 +147,7 @@ public class PTFOperator extends Operato
protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException {
PartitionDef pDef = conf.getStartOfChain().getPartition();
- ArrayList<PTFExpressionDef> exprs = pDef.getExpressions();
+ List<PTFExpressionDef> exprs = pDef.getExpressions();
int numExprs = exprs.size();
ExprNodeEvaluator[] keyFields = new ExprNodeEvaluator[numExprs];
ObjectInspector[] keyOIs = new ObjectInspector[numExprs];
@@ -220,7 +220,7 @@ public class PTFOperator extends Operato
*/
private PTFPartition executeChain(PTFPartition part)
throws HiveException {
- Stack<PartitionedTableFunctionDef> fnDefs = new Stack<PartitionedTableFunctionDef>();
+ Deque<PartitionedTableFunctionDef> fnDefs = new ArrayDeque<PartitionedTableFunctionDef>();
PTFInputDef iDef = conf.getFuncDef();
while (iDef instanceof PartitionedTableFunctionDef) {
@@ -289,6 +289,4 @@ public class PTFOperator extends Operato
}
}
-
-
}
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java Fri Oct 4 21:30:38 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.Text;
/**
@@ -72,11 +73,11 @@ public class ReduceSinkOperator extends
// TODO: we use MetadataTypedColumnsetSerDe for now, till DynamicSerDe is
// ready
- transient Serializer keySerializer;
- transient boolean keyIsText;
- transient Serializer valueSerializer;
+ protected transient Serializer keySerializer;
+ protected transient boolean keyIsText;
+ protected transient Serializer valueSerializer;
transient int tag;
- transient byte[] tagByte = new byte[1];
+ protected transient byte[] tagByte = new byte[1];
transient protected int numDistributionKeys;
transient protected int numDistinctExprs;
transient String inputAlias; // input alias of this RS for join (used for PPD)
@@ -163,12 +164,15 @@ public class ReduceSinkOperator extends
}
transient InspectableObject tempInspectableObject = new InspectableObject();
- transient HiveKey keyWritable = new HiveKey();
+ protected transient HiveKey keyWritable = new HiveKey();
+ protected transient Writable value;
transient StructObjectInspector keyObjectInspector;
transient StructObjectInspector valueObjectInspector;
transient ObjectInspector[] partitionObjectInspectors;
+ protected transient Object[] cachedValues;
+ protected transient List<List<Integer>> distinctColIndices;
/**
* This two dimensional array holds key data and a corresponding Union object
* which contains the tag identifying the aggregate expression for distinct columns.
@@ -183,13 +187,9 @@ public class ReduceSinkOperator extends
* in this case, child GBY evaluates distict values with expression like KEY.col2:0.dist1
* see {@link ExprNodeColumnEvaluator}
*/
- transient Object[][] cachedKeys;
- transient Object[] cachedValues;
- transient List<List<Integer>> distinctColIndices;
-
+ protected transient Object[][] cachedKeys;
boolean firstRow;
-
- transient Random random;
+ protected transient Random random;
/**
* Initializes array of ExprNodeEvaluator. Adds Union field for distinct
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java Fri Oct 4 21:30:38 2013
@@ -80,15 +80,20 @@ public class UnionOperator extends Opera
for (int p = 0; p < parents; p++) {
assert (parentFields[p].size() == columns);
for (int c = 0; c < columns; c++) {
- columnTypeResolvers[c].update(parentFields[p].get(c)
- .getFieldObjectInspector());
+ if (!columnTypeResolvers[c].update(parentFields[p].get(c)
+ .getFieldObjectInspector())) {
+ // checked in SemanticAnalyzer. Should not happen
+ throw new HiveException("Incompatible types for union operator");
+ }
}
}
ArrayList<ObjectInspector> outputFieldOIs = new ArrayList<ObjectInspector>(
columns);
for (int c = 0; c < columns; c++) {
- outputFieldOIs.add(columnTypeResolvers[c].get());
+ // can be null for void type
+ ObjectInspector oi = columnTypeResolvers[c].get();
+ outputFieldOIs.add(oi == null ? parentFields[0].get(c).getFieldObjectInspector() : oi);
}
// create output row ObjectInspector
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri Oct 4 21:30:38 2013
@@ -102,12 +102,12 @@ import org.apache.hadoop.hive.ql.Context
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryPlan;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.mr.ExecDriver;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
import org.apache.hadoop.hive.ql.exec.mr.ExecReducer;
import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
import org.apache.hadoop.hive.ql.io.ContentSummaryInputFormat;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
@@ -497,6 +497,18 @@ public final class Utilities {
}
}
+ /**
+ * Need to serialize org.antlr.runtime.CommonToken
+ */
+ public static class CommonTokenDelegate extends PersistenceDelegate {
+ @Override
+ protected Expression instantiate(Object oldInstance, Encoder out) {
+ CommonToken ct = (CommonToken)oldInstance;
+ Object[] args = {ct.getType(), ct.getText()};
+ return new Expression(ct, ct.getClass(), "new", args);
+ }
+ }
+
public static void setMapRedWork(Configuration conf, MapredWork w, String hiveScratchDir) {
setMapWork(conf, w.getMapWork(), hiveScratchDir, true);
if (w.getReduceWork() != null) {
@@ -569,7 +581,7 @@ public final class Utilities {
}
}
- private static Path getPlanPath(Configuration conf) {
+ public static Path getPlanPath(Configuration conf) {
String plan = HiveConf.getVar(conf, HiveConf.ConfVars.PLAN);
if (plan != null && !plan.isEmpty()) {
return new Path(plan);
@@ -746,7 +758,7 @@ public final class Utilities {
PerfLogger perfLogger = PerfLogger.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN);
ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
- Configuration conf = new Configuration();
+ Configuration conf = new HiveConf();
serializePlan(plan, baos, conf, true);
MapredWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()),
MapredWork.class, conf, true);
@@ -775,6 +787,7 @@ public final class Utilities {
e.setPersistenceDelegate(org.datanucleus.store.types.backed.Map.class, new MapDelegate());
e.setPersistenceDelegate(org.datanucleus.store.types.backed.List.class, new ListDelegate());
+ e.setPersistenceDelegate(CommonToken.class, new CommonTokenDelegate());
e.writeObject(plan);
e.close();
@@ -1694,7 +1707,7 @@ public final class Utilities {
for (String p : paths) {
Path path = new Path(p);
- RecordWriter writer = HiveFileFormatUtils.getRecordWriter(
+ FSRecordWriter writer = HiveFileFormatUtils.getRecordWriter(
jc, hiveOutputFormat, outputClass, isCompressed,
tableInfo.getProperties(), path, reporter);
writer.close(false);
@@ -2853,7 +2866,7 @@ public final class Utilities {
Path newFilePath = new Path(newFile);
String onefile = newPath.toString();
- RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newFilePath,
+ FSRecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newFilePath,
Text.class, false, props, null);
if (dummyRow) {
// empty files are omitted at CombineHiveInputFormat.
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java Fri Oct 4 21:30:38 2013
@@ -237,6 +237,7 @@ public class ExecDriver extends Task<Map
ShimLoader.getHadoopShims().prepareJobOutput(job);
//See the javadoc on HiveOutputFormatImpl and HadoopShims.prepareJobOutput()
job.setOutputFormat(HiveOutputFormatImpl.class);
+
job.setMapperClass(ExecMapper.class);
job.setMapOutputKeyClass(HiveKey.class);
@@ -828,3 +829,4 @@ public class ExecDriver extends Task<Map
}
}
}
+
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java Fri Oct 4 21:30:38 2013
@@ -28,8 +28,8 @@ import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.PTFDeserializer;
@@ -240,7 +240,7 @@ public class PTFRowContainer<Row extends
}
- private static class PTFRecordWriter implements RecordWriter {
+ private static class PTFRecordWriter implements FSRecordWriter {
BytesWritable EMPTY_KEY = new BytesWritable();
SequenceFile.Writer outStream;
@@ -262,7 +262,7 @@ public class PTFRowContainer<Row extends
extends HiveSequenceFileOutputFormat<K,V> {
@Override
- public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
+ public FSRecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
Class<? extends Writable> valueClass, boolean isCompressed,
Properties tableProperties, Progressable progress) throws IOException {
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java Fri Oct 4 21:30:38 2013
@@ -30,8 +30,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -105,7 +105,7 @@ public class RowContainer<ROW extends Li
int acutalSplitNum = 0;
int currentSplitPointer = 0;
org.apache.hadoop.mapred.RecordReader rr = null; // record reader
- RecordWriter rw = null;
+ FSRecordWriter rw = null;
InputFormat<WritableComparable, Writable> inputFormat = null;
InputSplit[] inputSplits = null;
private ROW dummyRow = null;
@@ -531,7 +531,7 @@ public class RowContainer<ROW extends Li
}
- protected RecordWriter getRecordWriter() {
+ protected FSRecordWriter getRecordWriter() {
return rw;
}
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java Fri Oct 4 21:30:38 2013
@@ -24,7 +24,6 @@ import java.util.Properties;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
@@ -43,7 +42,7 @@ public class HiveBinaryOutputFormat<K ex
/**
* create the final out file, and output row by row. After one row is
* appended, a configured row separator is appended
- *
+ *
* @param jc
* the job configuration file
* @param outPath
@@ -59,14 +58,14 @@ public class HiveBinaryOutputFormat<K ex
* @return the RecordWriter
*/
@Override
- public RecordWriter getHiveRecordWriter(JobConf jc, Path outPath,
+ public FSRecordWriter getHiveRecordWriter(JobConf jc, Path outPath,
Class<? extends Writable> valueClass, boolean isCompressed,
Properties tableProperties, Progressable progress) throws IOException {
FileSystem fs = outPath.getFileSystem(jc);
final OutputStream outStream = fs.create(outPath);
- return new RecordWriter() {
+ return new FSRecordWriter() {
public void write(Writable r) throws IOException {
if (r instanceof Text) {
Text tr = (Text) r;
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java Fri Oct 4 21:30:38 2013
@@ -32,7 +32,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat;
@@ -246,7 +245,7 @@ public final class HiveFileFormatUtils {
return true;
}
- public static RecordWriter getHiveRecordWriter(JobConf jc,
+ public static FSRecordWriter getHiveRecordWriter(JobConf jc,
TableDesc tableInfo, Class<? extends Writable> outputClass,
FileSinkDesc conf, Path outPath, Reporter reporter) throws HiveException {
boolean storagehandlerofhivepassthru = false;
@@ -287,7 +286,7 @@ public final class HiveFileFormatUtils {
}
}
- public static RecordWriter getRecordWriter(JobConf jc,
+ public static FSRecordWriter getRecordWriter(JobConf jc,
HiveOutputFormat<?, ?> hiveOutputFormat,
final Class<? extends Writable> valueClass, boolean isCompressed,
Properties tableProp, Path outPath, Reporter reporter
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java Fri Oct 4 21:30:38 2013
@@ -25,7 +25,6 @@ import java.util.Properties;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
@@ -39,7 +38,7 @@ import org.apache.hadoop.util.Progressab
/**
* HiveIgnoreKeyTextOutputFormat replaces key with null before feeding the <key,
* value> to TextOutputFormat.RecordWriter.
- *
+ *
*/
public class HiveIgnoreKeyTextOutputFormat<K extends WritableComparable, V extends Writable>
extends TextOutputFormat<K, V> implements HiveOutputFormat<K, V> {
@@ -47,7 +46,7 @@ public class HiveIgnoreKeyTextOutputForm
/**
* create the final out file, and output row by row. After one row is
* appended, a configured row separator is appended
- *
+ *
* @param jc
* the job configuration file
* @param outPath
@@ -63,7 +62,7 @@ public class HiveIgnoreKeyTextOutputForm
* @return the RecordWriter
*/
@Override
- public RecordWriter getHiveRecordWriter(JobConf jc, Path outPath,
+ public FSRecordWriter getHiveRecordWriter(JobConf jc, Path outPath,
Class<? extends Writable> valueClass, boolean isCompressed,
Properties tableProperties, Progressable progress) throws IOException {
int rowSeparator = 0;
@@ -79,7 +78,7 @@ public class HiveIgnoreKeyTextOutputForm
FileSystem fs = outPath.getFileSystem(jc);
final OutputStream outStream = Utilities.createCompressedStream(jc, fs
.create(outPath), isCompressed);
- return new RecordWriter() {
+ return new FSRecordWriter() {
public void write(Writable r) throws IOException {
if (r instanceof Text) {
Text tr = (Text) r;
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java Fri Oct 4 21:30:38 2013
@@ -23,7 +23,6 @@ import java.util.Properties;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
@@ -48,7 +47,7 @@ public class HiveNullValueSequenceFileOu
private boolean keyIsText;
@Override
- public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
+ public FSRecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
Class<? extends Writable> valueClass, boolean isCompressed,
Properties tableProperties, Progressable progress) throws IOException {
@@ -58,7 +57,7 @@ public class HiveNullValueSequenceFileOu
keyWritable = new HiveKey();
keyIsText = valueClass.equals(Text.class);
- return new RecordWriter() {
+ return new FSRecordWriter() {
public void write(Writable r) throws IOException {
if (keyIsText) {
Text text = (Text) r;
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormat.java Fri Oct 4 21:30:38 2013
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.Properties;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;
@@ -58,7 +57,7 @@ public interface HiveOutputFormat<K, V>
* progress used for status report
* @return the RecordWriter for the output file
*/
- RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
+ FSRecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
final Class<? extends Writable> valueClass, boolean isCompressed,
Properties tableProperties, Progressable progress) throws IOException;
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughOutputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughOutputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughOutputFormat.java Fri Oct 4 21:30:38 2013
@@ -49,7 +49,7 @@ public class HivePassThroughOutputFormat
"org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat";
public static final String HIVE_PASSTHROUGH_STORAGEHANDLER_OF_JOBCONFKEY =
- "hive.passthrough.storagehandler.of";
+ "hive.passthrough.storagehandler.of";
public HivePassThroughOutputFormat() {
//construct this class through ReflectionUtils from FileSinkOperator
@@ -99,7 +99,7 @@ public class HivePassThroughOutputFormat
}
@Override
- public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter(
+ public FSRecordWriter getHiveRecordWriter(
JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed,
Properties tableProperties, Progressable progress) throws IOException {
if (this.initialized == false) {
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java Fri Oct 4 21:30:38 2013
@@ -20,13 +20,12 @@ package org.apache.hadoop.hive.ql.io;
import java.io.IOException;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
public class HivePassThroughRecordWriter <K extends WritableComparable<?>, V extends Writable>
-implements RecordWriter {
+implements FSRecordWriter {
private final org.apache.hadoop.mapred.RecordWriter<K, V> mWriter;
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java Fri Oct 4 21:30:38 2013
@@ -23,7 +23,6 @@ import java.util.Properties;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.SequenceFile;
@@ -56,7 +55,7 @@ public class HiveSequenceFileOutputForma
* @return the RecordWriter for the output file
*/
@Override
- public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
+ public FSRecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
Class<? extends Writable> valueClass, boolean isCompressed,
Properties tableProperties, Progressable progress) throws IOException {
@@ -64,7 +63,7 @@ public class HiveSequenceFileOutputForma
final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
fs, finalOutPath, BytesWritable.class, valueClass, isCompressed);
- return new RecordWriter() {
+ return new FSRecordWriter() {
public void write(Writable r) throws IOException {
outStream.append(EMPTY_KEY, r);
}
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java Fri Oct 4 21:30:38 2013
@@ -118,7 +118,7 @@ public class RCFileOutputFormat extends
* @throws IOException
*/
@Override
- public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter(
+ public FSRecordWriter getHiveRecordWriter(
JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass,
boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException {
@@ -135,7 +135,7 @@ public class RCFileOutputFormat extends
(jc, finalOutPath.getFileSystem(jc),
finalOutPath, isCompressed);
- return new org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter() {
+ return new FSRecordWriter() {
public void write(Writable r) throws IOException {
outWriter.append(r);
}
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java Fri Oct 4 21:30:38 2013
@@ -17,6 +17,14 @@
*/
package org.apache.hadoop.hive.ql.io.avro;
+import static org.apache.avro.file.DataFileConstants.DEFLATE_CODEC;
+import static org.apache.avro.mapred.AvroJob.OUTPUT_CODEC;
+import static org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL;
+import static org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY;
+
+import java.io.IOException;
+import java.util.Properties;
+
import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileWriter;
@@ -24,7 +32,7 @@ import org.apache.avro.generic.GenericDa
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable;
import org.apache.hadoop.hive.serde2.avro.AvroSerdeException;
@@ -36,14 +44,6 @@ import org.apache.hadoop.mapred.RecordWr
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
-import java.io.IOException;
-import java.util.Properties;
-
-import static org.apache.avro.file.DataFileConstants.DEFLATE_CODEC;
-import static org.apache.avro.mapred.AvroJob.OUTPUT_CODEC;
-import static org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL;
-import static org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY;
-
/**
* Write to an Avro file from a Hive process.
*/
@@ -51,7 +51,7 @@ public class AvroContainerOutputFormat
implements HiveOutputFormat<LongWritable, AvroGenericRecordWritable> {
@Override
- public FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf,
+ public FSRecordWriter getHiveRecordWriter(JobConf jobConf,
Path path, Class<? extends Writable> valueClass, boolean isCompressed,
Properties properties, Progressable progressable) throws IOException {
Schema schema;
@@ -62,7 +62,7 @@ public class AvroContainerOutputFormat
}
GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema);
DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);
-
+
if (isCompressed) {
int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC);
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordWriter.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordWriter.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordWriter.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordWriter.java Fri Oct 4 21:30:38 2013
@@ -18,18 +18,18 @@
package org.apache.hadoop.hive.ql.io.avro;
+import java.io.IOException;
+
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable;
import org.apache.hadoop.io.Writable;
-import java.io.IOException;
-
/**
* Write an Avro GenericRecord to an Avro data file.
*/
-public class AvroGenericRecordWriter implements FileSinkOperator.RecordWriter{
+public class AvroGenericRecordWriter implements FSRecordWriter{
final private DataFileWriter<GenericRecord> dfw;
public AvroGenericRecordWriter(DataFileWriter<GenericRecord> dfw) {
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java Fri Oct 4 21:30:38 2013
@@ -20,8 +20,10 @@ package org.apache.hadoop.hive.ql.io.orc
import java.io.EOFException;
import java.io.IOException;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+
class BitFieldReader {
- private RunLengthByteReader input;
+ private final RunLengthByteReader input;
private final int bitSize;
private int current;
private int bitsLeft;
@@ -60,6 +62,30 @@ class BitFieldReader {
return result & mask;
}
+ void nextVector(LongColumnVector previous, long previousLen)
+ throws IOException {
+
+ previous.isRepeating = true;
+ for (int i = 0; i < previousLen; i++) {
+ if (!previous.isNull[i]) {
+ previous.vector[i] = next();
+ } else {
+ // The default value of null for int types in vectorized
+ // processing is 1, so set that if the value is null
+ previous.vector[i] = 1;
+ }
+
+ // The default value for nulls in Vectorization for int types is 1
+ // and given that non null value can also be 1, we need to check for isNull also
+ // when determining the isRepeating flag.
+ if (previous.isRepeating
+ && i > 0
+ && ((previous.vector[i - 1] != previous.vector[i]) || (previous.isNull[i - 1] != previous.isNull[i]))) {
+ previous.isRepeating = false;
+ }
+ }
+ }
+
void seek(PositionProvider index) throws IOException {
input.seek(index);
int consumed = (int) index.getNext();
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java Fri Oct 4 21:30:38 2013
@@ -21,6 +21,7 @@ import org.apache.hadoop.hive.common.typ
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.io.BytesWritable;
class ColumnStatisticsImpl implements ColumnStatistics {
@@ -332,10 +333,11 @@ class ColumnStatisticsImpl implements Co
}
}
- private static final class StringStatisticsImpl extends ColumnStatisticsImpl
+ protected static final class StringStatisticsImpl extends ColumnStatisticsImpl
implements StringColumnStatistics {
private String minimum = null;
private String maximum = null;
+ private long sum = 0;
StringStatisticsImpl() {
}
@@ -349,6 +351,9 @@ class ColumnStatisticsImpl implements Co
if (str.hasMinimum()) {
minimum = str.getMinimum();
}
+ if(str.hasSum()) {
+ sum = str.getSum();
+ }
}
@Override
@@ -356,6 +361,7 @@ class ColumnStatisticsImpl implements Co
super.reset();
minimum = null;
maximum = null;
+ sum = 0;
}
@Override
@@ -368,6 +374,7 @@ class ColumnStatisticsImpl implements Co
} else if (maximum.compareTo(value) < 0) {
maximum = value;
}
+ sum += value.length();
}
@Override
@@ -384,6 +391,7 @@ class ColumnStatisticsImpl implements Co
maximum = str.maximum;
}
}
+ sum += str.sum;
}
@Override
@@ -394,6 +402,7 @@ class ColumnStatisticsImpl implements Co
if (getNumberOfValues() != 0) {
str.setMinimum(minimum);
str.setMaximum(maximum);
+ str.setSum(sum);
}
result.setStringStatistics(str);
return result;
@@ -410,6 +419,11 @@ class ColumnStatisticsImpl implements Co
}
@Override
+ public long getSum() {
+ return sum;
+ }
+
+ @Override
public String toString() {
StringBuilder buf = new StringBuilder(super.toString());
if (getNumberOfValues() != 0) {
@@ -417,6 +431,67 @@ class ColumnStatisticsImpl implements Co
buf.append(minimum);
buf.append(" max: ");
buf.append(maximum);
+ buf.append(" sum: ");
+ buf.append(sum);
+ }
+ return buf.toString();
+ }
+ }
+
+ protected static final class BinaryStatisticsImpl extends ColumnStatisticsImpl implements
+ BinaryColumnStatistics {
+
+ private long sum = 0;
+
+ BinaryStatisticsImpl() {
+ }
+
+ BinaryStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ super(stats);
+ OrcProto.BinaryStatistics binStats = stats.getBinaryStatistics();
+ if (binStats.hasSum()) {
+ sum = binStats.getSum();
+ }
+ }
+
+ @Override
+ void reset() {
+ super.reset();
+ sum = 0;
+ }
+
+ @Override
+ void updateBinary(BytesWritable value) {
+ sum += value.getLength();
+ }
+
+ @Override
+ void merge(ColumnStatisticsImpl other) {
+ super.merge(other);
+ BinaryStatisticsImpl bin = (BinaryStatisticsImpl) other;
+ sum += bin.sum;
+ }
+
+ @Override
+ public long getSum() {
+ return sum;
+ }
+
+ @Override
+ OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder result = super.serialize();
+ OrcProto.BinaryStatistics.Builder bin = OrcProto.BinaryStatistics.newBuilder();
+ bin.setSum(sum);
+ result.setBinaryStatistics(bin);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (getNumberOfValues() != 0) {
+ buf.append(" sum: ");
+ buf.append(sum);
}
return buf.toString();
}
@@ -666,6 +741,10 @@ class ColumnStatisticsImpl implements Co
throw new UnsupportedOperationException("Can't update string");
}
+ void updateBinary(BytesWritable value) {
+ throw new UnsupportedOperationException("Can't update binary");
+ }
+
void updateDecimal(HiveDecimal value) {
throw new UnsupportedOperationException("Can't update decimal");
}
@@ -720,6 +799,8 @@ class ColumnStatisticsImpl implements Co
return new DecimalStatisticsImpl();
case DATE:
return new DateStatisticsImpl();
+ case BINARY:
+ return new BinaryStatisticsImpl();
default:
return new ColumnStatisticsImpl();
}
@@ -741,6 +822,8 @@ class ColumnStatisticsImpl implements Co
return new DecimalStatisticsImpl(stats);
} else if (stats.hasDateStatistics()) {
return new DateStatisticsImpl(stats);
+ } else if(stats.hasBinaryStatistics()) {
+ return new BinaryStatisticsImpl(stats);
} else {
return new ColumnStatisticsImpl(stats);
}
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java Fri Oct 4 21:30:38 2013
@@ -17,13 +17,13 @@
*/
package org.apache.hadoop.hive.ql.io.orc;
-import org.apache.hadoop.io.Text;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import org.apache.hadoop.io.Text;
+
/**
* A class that is a growable array of bytes. Growth is managed in terms of
* chunks that are allocated when needed.
@@ -237,6 +237,7 @@ final class DynamicByteArray {
}
}
+ @Override
public String toString() {
int i;
StringBuilder sb = new StringBuilder(length * 3);
@@ -268,10 +269,35 @@ final class DynamicByteArray {
}
/**
+ * Gets all the bytes of the array.
+ *
+ * @return Bytes of the array
+ */
+ public byte[] get() {
+ byte[] result = null;
+ if (length > 0) {
+ int currentChunk = 0;
+ int currentOffset = 0;
+ int currentLength = Math.min(length, chunkSize);
+ int destOffset = 0;
+ result = new byte[length];
+ int totalLength = length;
+ while (totalLength > 0) {
+ System.arraycopy(data[currentChunk], currentOffset, result, destOffset, currentLength);
+ destOffset += currentLength;
+ totalLength -= currentLength;
+ currentChunk += 1;
+ currentOffset = 0;
+ currentLength = Math.min(totalLength, chunkSize - currentOffset);
+ }
+ }
+ return result;
+ }
+
+ /**
* Get the size of the buffers.
*/
public long getSizeInBytes() {
return initializedChunks * chunkSize;
}
}
-
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerReader.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerReader.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerReader.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerReader.java Fri Oct 4 21:30:38 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.io.orc;
import java.io.IOException;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
/**
* Interface for reading integers.
@@ -52,4 +53,12 @@ interface IntegerReader {
* @throws IOException
*/
long next() throws IOException;
+
+ /**
+ * Return the next available vector for values.
+ * @return
+ * @throws IOException
+ */
+ void nextVector(LongColumnVector previous, long previousLen)
+ throws IOException;
}
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Fri Oct 4 21:30:38 2013
@@ -27,7 +27,6 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -38,6 +37,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.InputFormatChecker;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
@@ -54,12 +55,13 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.StringUtils;
-
/**
* A MapReduce/Hive input format for ORC files.
*/
-public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
- InputFormatChecker {
+public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
+ InputFormatChecker, VectorizedInputFormatInterface {
+
+ VectorizedOrcInputFormat voif = new VectorizedOrcInputFormat();
private static final Log LOG = LogFactory.getLog(OrcInputFormat.class);
static final String MIN_SPLIT_SIZE = "mapred.min.split.size";
@@ -85,6 +87,7 @@ public class OrcInputFormat implements I
private final int numColumns;
private float progress = 0.0f;
+
OrcRecordReader(Reader file, Configuration conf,
long offset, long length) throws IOException {
String serializedPushdown = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
@@ -111,7 +114,7 @@ public class OrcInputFormat implements I
String[] neededColumnNames = columnNamesString.split(",");
int i = 0;
for(int columnId: types.get(0).getSubtypesList()) {
- if (includeColumn[columnId]) {
+ if (includeColumn == null || includeColumn[columnId]) {
columnNames[columnId] = neededColumnNames[i++];
}
}
@@ -219,6 +222,12 @@ public class OrcInputFormat implements I
public RecordReader<NullWritable, OrcStruct>
getRecordReader(InputSplit inputSplit, JobConf conf,
Reporter reporter) throws IOException {
+ if (isVectorMode(conf)) {
+ RecordReader<NullWritable, VectorizedRowBatch> vorr = voif.getRecordReader(inputSplit, conf,
+ reporter);
+ return (RecordReader) vorr;
+ }
+
FileSplit fileSplit = (FileSplit) inputSplit;
Path path = fileSplit.getPath();
FileSystem fs = path.getFileSystem(conf);
@@ -231,6 +240,11 @@ public class OrcInputFormat implements I
public boolean validateInput(FileSystem fs, HiveConf conf,
ArrayList<FileStatus> files
) throws IOException {
+
+ if (isVectorMode(conf)) {
+ return voif.validateInput(fs, conf, files);
+ }
+
if (files.size() <= 0) {
return false;
}
@@ -244,6 +258,14 @@ public class OrcInputFormat implements I
return true;
}
+ private boolean isVectorMode(Configuration conf) {
+ if (Utilities.getPlanPath(conf) != null && Utilities
+ .getMapRedWork(conf).getMapWork().getVectorMode()) {
+ return true;
+ }
+ return false;
+ }
+
/**
* Get the list of input {@link Path}s for the map-reduce job.
*
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java Fri Oct 4 21:30:38 2013
@@ -17,12 +17,12 @@
*/
package org.apache.hadoop.hive.ql.io.orc;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcSerde.OrcSerdeRow;
+import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.io.NullWritable;
@@ -45,14 +45,17 @@ public class OrcOutputFormat extends Fil
private static class OrcRecordWriter
implements RecordWriter<NullWritable, OrcSerdeRow>,
- FileSinkOperator.RecordWriter {
+ FSRecordWriter,
+ FSRecordWriter.StatsProvidingRecordWriter {
private Writer writer = null;
private final Path path;
private final OrcFile.WriterOptions options;
+ private final SerDeStats stats;
OrcRecordWriter(Path path, OrcFile.WriterOptions options) {
this.path = path;
this.options = options;
+ this.stats = new SerDeStats();
}
@Override
@@ -94,6 +97,13 @@ public class OrcOutputFormat extends Fil
}
writer.close();
}
+
+ @Override
+ public SerDeStats getStats() {
+ stats.setRawDataSize(writer.getRawDataSize());
+ stats.setRowCount(writer.getNumberOfRows());
+ return stats;
+ }
}
@Override
@@ -105,7 +115,7 @@ public class OrcOutputFormat extends Fil
}
@Override
- public FileSinkOperator.RecordWriter
+ public FSRecordWriter
getHiveRecordWriter(JobConf conf,
Path path,
Class<? extends Writable> valueClass,
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java Fri Oct 4 21:30:38 2013
@@ -17,9 +17,16 @@
*/
package org.apache.hadoop.hive.ql.io.orc;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedSerde;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeStats;
@@ -29,26 +36,22 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.Writable;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Properties;
-
/**
* A serde class for ORC.
* It transparently passes the object to/from the ORC file reader/writer.
*/
-public class OrcSerde implements SerDe {
+public class OrcSerde implements SerDe, VectorizedSerde {
private static final Log LOG = LogFactory.getLog(OrcSerde.class);
private final OrcSerdeRow row = new OrcSerdeRow();
private ObjectInspector inspector = null;
+ private VectorizedOrcSerde vos = null;
+
final class OrcSerdeRow implements Writable {
- private Object realRow;
- private ObjectInspector inspector;
+ Object realRow;
+ ObjectInspector inspector;
@Override
public void write(DataOutput dataOutput) throws IOException {
@@ -79,7 +82,7 @@ public class OrcSerde implements SerDe {
// Parse the configuration parameters
ArrayList<String> columnNames = new ArrayList<String>();
if (columnNameProperty != null && columnNameProperty.length() > 0) {
- for(String name: columnNameProperty.split(",")) {
+ for (String name : columnNameProperty.split(",")) {
columnNames.add(name);
}
}
@@ -96,7 +99,7 @@ public class OrcSerde implements SerDe {
}
ArrayList<TypeInfo> fieldTypes =
- TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
+ TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
StructTypeInfo rootType = new StructTypeInfo();
rootType.setAllStructFieldNames(columnNames);
rootType.setAllStructFieldTypeInfos(fieldTypes);
@@ -128,6 +131,7 @@ public class OrcSerde implements SerDe {
/**
* Always returns null, since serialized size doesn't make sense in the
* context of ORC files.
+ *
* @return null
*/
@Override
@@ -135,4 +139,18 @@ public class OrcSerde implements SerDe {
return null;
}
+ @Override
+ public Writable serializeVector(VectorizedRowBatch vrg, ObjectInspector objInspector)
+ throws SerDeException {
+ if (vos == null) {
+ vos = new VectorizedOrcSerde(objInspector);
+ }
+ return vos.serialize(vrg, objInspector);
+ }
+
+ @Override
+ public void deserializeVector(Object rowBlob, int rowsInBatch, VectorizedRowBatch reuseBatch)
+ throws SerDeException {
+ // nothing to do here
+ }
}
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java Fri Oct 4 21:30:38 2013
@@ -39,6 +39,19 @@ public interface Reader {
long getNumberOfRows();
/**
+ * Get the deserialized data size of the file
+ * @return raw data size
+ */
+ long getRawDataSize();
+
+ /**
+ * Get the deserialized data size of the specified columns
+ * @param colNames
+ * @return raw data size of columns
+ */
+ long getRawDataSizeOfColumns(List<String> colNames);
+
+ /**
* Get the user metadata keys.
* @return the set of metadata keys
*/
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java Fri Oct 4 21:30:38 2013
@@ -18,23 +18,29 @@
package org.apache.hadoop.hive.ql.io.orc;
-import com.google.protobuf.CodedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.util.StringUtils;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.protobuf.CodedInputStream;
final class ReaderImpl implements Reader {
@@ -49,6 +55,7 @@ final class ReaderImpl implements Reader
private final int bufferSize;
private final OrcProto.Footer footer;
private final ObjectInspector inspector;
+ private long deserializedSize = -1;
private static class StripeInformationImpl
implements StripeInformation {
@@ -343,4 +350,124 @@ final class ReaderImpl implements Reader
include, footer.getRowIndexStride(), sarg, columnNames);
}
+ @Override
+ public long getRawDataSize() {
+ // if the deserializedSize is not computed, then compute it, else
+ // return the already computed size. since we are reading from the footer
+ // we don't have to compute deserialized size repeatedly
+ if (deserializedSize == -1) {
+ List<OrcProto.ColumnStatistics> stats = footer.getStatisticsList();
+ List<Integer> indices = Lists.newArrayList();
+ for (int i = 0; i < stats.size(); ++i) {
+ indices.add(i);
+ }
+ deserializedSize = getRawDataSizeFromColIndices(indices);
+ }
+ return deserializedSize;
+ }
+
+ private long getRawDataSizeFromColIndices(List<Integer> colIndices) {
+ long result = 0;
+ for (int colIdx : colIndices) {
+ result += getRawDataSizeOfColumn(colIdx);
+ }
+ return result;
+ }
+
+ private long getRawDataSizeOfColumn(int colIdx) {
+ OrcProto.ColumnStatistics colStat = footer.getStatistics(colIdx);
+ long numVals = colStat.getNumberOfValues();
+ Type type = footer.getTypes(colIdx);
+
+ switch (type.getKind()) {
+ case BINARY:
+ // old orc format doesn't support binary statistics. checking for binary
+ // statistics is not required as protocol buffers takes care of it.
+ return colStat.getBinaryStatistics().getSum();
+ case STRING:
+ // old orc format doesn't support sum for string statistics. checking for
+ // existence is not required as protocol buffers takes care of it.
+
+ // ORC strings are deserialized to java strings. so use java data model's
+ // string size
+ numVals = numVals == 0 ? 1 : numVals;
+ int avgStrLen = (int) (colStat.getStringStatistics().getSum() / numVals);
+ return numVals * JavaDataModel.get().lengthForStringOfLength(avgStrLen);
+ case TIMESTAMP:
+ return numVals * JavaDataModel.get().lengthOfTimestamp();
+ case DATE:
+ return numVals * JavaDataModel.get().lengthOfDate();
+ case DECIMAL:
+ return numVals * JavaDataModel.get().lengthOfDecimal();
+ case DOUBLE:
+ case LONG:
+ return numVals * JavaDataModel.get().primitive2();
+ case FLOAT:
+ case INT:
+ case SHORT:
+ case BOOLEAN:
+ case BYTE:
+ return numVals * JavaDataModel.get().primitive1();
+ default:
+ LOG.debug("Unknown primitive category.");
+ break;
+ }
+
+ return 0;
+ }
+
+ @Override
+ public long getRawDataSizeOfColumns(List<String> colNames) {
+ List<Integer> colIndices = getColumnIndicesFromNames(colNames);
+ return getRawDataSizeFromColIndices(colIndices);
+ }
+
+ private List<Integer> getColumnIndicesFromNames(List<String> colNames) {
+ // top level struct
+ Type type = footer.getTypesList().get(0);
+ List<Integer> colIndices = Lists.newArrayList();
+ List<String> fieldNames = type.getFieldNamesList();
+ int fieldIdx = 0;
+ for (String colName : colNames) {
+ if (fieldNames.contains(colName)) {
+ fieldIdx = fieldNames.indexOf(colName);
+ }
+
+ // a single field may span multiple columns. find start and end column
+ // index for the requested field
+ int idxStart = type.getSubtypes(fieldIdx);
+
+ int idxEnd = 0;
+
+ // if the specified is the last field and then end index will be last
+ // column index
+ if (fieldIdx + 1 > fieldNames.size() - 1) {
+ idxEnd = getLastIdx() + 1;
+ } else {
+ idxEnd = type.getSubtypes(fieldIdx + 1);
+ }
+
+ // if start index and end index are same then the field is a primitive
+ // field else complex field (like map, list, struct, union)
+ if (idxStart == idxEnd) {
+ // simple field
+ colIndices.add(idxStart);
+ } else {
+ // complex fields spans multiple columns
+ for (int i = idxStart; i < idxEnd; i++) {
+ colIndices.add(i);
+ }
+ }
+ }
+ return colIndices;
+ }
+
+ private int getLastIdx() {
+ Set<Integer> indices = Sets.newHashSet();
+ for (Type type : footer.getTypesList()) {
+ indices.addAll(type.getSubtypesList());
+ }
+ return Collections.max(indices);
+ }
+
}
Modified: hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java?rev=1529308&r1=1529307&r2=1529308&view=diff
==============================================================================
--- hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java (original)
+++ hive/branches/maven/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java Fri Oct 4 21:30:38 2013
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.io.orc
import java.io.IOException;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
/**
* A row-by-row iterator for ORC files.
*/
@@ -39,6 +41,16 @@ public interface RecordReader {
Object next(Object previous) throws IOException;
/**
+ * Read the next row batch. The size of the batch to read cannot be controlled
+ * by the callers. Caller need to look at VectorizedRowBatch.size of the retunred
+ * object to know the batch size read.
+ * @param previousBatch a row batch object that can be reused by the reader
+ * @return the row batch that was read
+ * @throws java.io.IOException
+ */
+ VectorizedRowBatch nextBatch(VectorizedRowBatch previousBatch) throws IOException;
+
+ /**
* Get the row number of the row that will be returned by the following
* call to next().
* @return the row number from 0 to the number of rows in the file