You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by su...@apache.org on 2015/07/20 22:12:50 UTC
[40/50] [abbrv] hive git commit: HIVE-11080. Make
VectorizedRowBatch.toString not depend on VectorExpressionWriter.
HIVE-11080. Make VectorizedRowBatch.toString not depend on
VectorExpressionWriter.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b6f48cb9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b6f48cb9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b6f48cb9
Branch: refs/heads/spark
Commit: b6f48cb915b7e691b77cc4f328b9a32a082aa052
Parents: 178b8d1
Author: Owen O'Malley <om...@apache.org>
Authored: Thu Jun 25 11:21:09 2015 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Jul 17 09:43:52 2015 -0700
----------------------------------------------------------------------
.../ql/exec/spark/SparkReduceRecordHandler.java | 2 -
.../hive/ql/exec/tez/ReduceRecordSource.java | 3 -
.../hive/ql/exec/vector/BytesColumnVector.java | 14 ++++
.../hive/ql/exec/vector/ColumnVector.java | 8 ++
.../ql/exec/vector/DecimalColumnVector.java | 12 +++
.../hive/ql/exec/vector/DoubleColumnVector.java | 12 +++
.../hive/ql/exec/vector/LongColumnVector.java | 12 +++
.../ql/exec/vector/VectorSelectOperator.java | 3 -
.../hive/ql/exec/vector/VectorizedRowBatch.java | 78 ++++++++------------
9 files changed, 87 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b6f48cb9/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
index ac5e3ca..bedccc3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
@@ -364,8 +364,6 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
} catch (Exception e) {
String rowString = null;
try {
- /* batch.toString depends on this */
- batch.setValueWriters(valueStringWriters[tag].toArray(new VectorExpressionWriter[0]));
rowString = batch.toString();
} catch (Exception e2) {
rowString = "[Error getting row data with exception " + StringUtils.stringifyException(e2)
http://git-wip-us.apache.org/repos/asf/hive/blob/b6f48cb9/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
index 7d79e87..20f6dba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
@@ -444,9 +444,6 @@ public class ReduceRecordSource implements RecordSource {
} catch (Exception e) {
String rowString = null;
try {
- /* batch.toString depends on this */
- batch.setValueWriters(valueStringWriters
- .toArray(new VectorExpressionWriter[0]));
rowString = batch.toString();
} catch (Exception e2) {
rowString = "[Error getting row data with exception "
http://git-wip-us.apache.org/repos/asf/hive/blob/b6f48cb9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
index 8ec7ead..c9a0fa2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
@@ -330,4 +330,18 @@ public class BytesColumnVector extends ColumnVector {
public void init() {
initBuffer(0);
}
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append('"');
+ buffer.append(new String(this.buffer, start[row], length[row]));
+ buffer.append('"');
+ } else {
+ buffer.append("null");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b6f48cb9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
index 6654166..49d4c12 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
@@ -166,5 +166,13 @@ public abstract class ColumnVector {
public void init() {
// Do nothing by default
}
+
+ /**
+ * Print the value for this column into the given string builder.
+ * @param buffer the buffer to print into
+ * @param row the id of the row to print
+ */
+ public abstract void stringifyValue(StringBuilder buffer,
+ int row);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b6f48cb9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
index 5009a42..0f63b29 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
@@ -85,6 +85,18 @@ public class DecimalColumnVector extends ColumnVector {
}
}
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append(vector[row].toString());
+ } else {
+ buffer.append("null");
+ }
+ }
+
public void set(int elementNum, HiveDecimalWritable writeable) {
HiveDecimal hiveDec = writeable.getHiveDecimal(precision, scale);
if (hiveDec == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b6f48cb9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java
index 525b3c5..013a9f5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java
@@ -146,4 +146,16 @@ public class DoubleColumnVector extends ColumnVector {
public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
vector[outElementNum] = ((DoubleColumnVector) inputVector).vector[inputElementNum];
}
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append(vector[row]);
+ } else {
+ buffer.append("null");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b6f48cb9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java
index f0545fe..d900cc6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java
@@ -190,4 +190,16 @@ public class LongColumnVector extends ColumnVector {
public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
vector[outElementNum] = ((LongColumnVector) inputVector).vector[inputElementNum];
}
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append(vector[row]);
+ } else {
+ buffer.append("null");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b6f48cb9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
index 212aa99..86b74e7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
@@ -129,8 +129,6 @@ public class VectorSelectOperator extends Operator<SelectDesc> implements
}
// Prepare output, set the projections
- VectorExpressionWriter [] originalValueWriters = vrg.valueWriters;
- vrg.setValueWriters(valueWriters);
int[] originalProjections = vrg.projectedColumns;
int originalProjectionSize = vrg.projectionSize;
vrg.projectionSize = vExpressions.length;
@@ -140,7 +138,6 @@ public class VectorSelectOperator extends Operator<SelectDesc> implements
// Revert the projected columns back, because vrg will be re-used.
vrg.projectionSize = originalProjectionSize;
vrg.projectedColumns = originalProjections;
- vrg.valueWriters = originalValueWriters;
}
static public String getOperatorName() {
http://git-wip-us.apache.org/repos/asf/hive/blob/b6f48cb9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
index 7e41384..7c18da6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
@@ -20,13 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
@@ -61,8 +55,6 @@ public class VectorizedRowBatch implements Writable {
*/
public static final int DEFAULT_SIZE = 1024;
- public VectorExpressionWriter[] valueWriters = null;
-
/**
* Return a batch with the specified number of columns.
* This is the standard constructor -- all batches should be the same size
@@ -112,7 +104,7 @@ public class VectorizedRowBatch implements Writable {
return size;
}
- private String toUTF8(Object o) {
+ private static String toUTF8(Object o) {
if(o == null || o instanceof NullWritable) {
return "\\N"; /* as found in LazySimpleSerDe's nullSequence */
}
@@ -125,47 +117,39 @@ public class VectorizedRowBatch implements Writable {
return "";
}
StringBuilder b = new StringBuilder();
- try {
- if (this.selectedInUse) {
- for (int j = 0; j < size; j++) {
- int i = selected[j];
- for (int k = 0; k < projectionSize; k++) {
- int projIndex = projectedColumns[k];
- ColumnVector cv = cols[projIndex];
- if (k > 0) {
- b.append('\u0001');
- }
- if (cv.isRepeating) {
- b.append(toUTF8(valueWriters[k].writeValue(cv, 0)));
- } else {
- b.append(toUTF8(valueWriters[k].writeValue(cv, i)));
- }
- }
- if (j < size - 1) {
- b.append('\n');
+ if (this.selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = selected[j];
+ b.append('[');
+ for (int k = 0; k < projectionSize; k++) {
+ int projIndex = projectedColumns[k];
+ ColumnVector cv = cols[projIndex];
+ if (k > 0) {
+ b.append(", ");
}
+ cv.stringifyValue(b, i);
}
- } else {
- for (int i = 0; i < size; i++) {
- for (int k = 0; k < projectionSize; k++) {
- int projIndex = projectedColumns[k];
- ColumnVector cv = cols[projIndex];
- if (k > 0) {
- b.append('\u0001');
- }
- if (cv.isRepeating) {
- b.append(toUTF8(valueWriters[k].writeValue(cv, 0)));
- } else {
- b.append(toUTF8(valueWriters[k].writeValue(cv, i)));
- }
- }
- if (i < size - 1) {
- b.append('\n');
+ b.append(']');
+ if (j < size - 1) {
+ b.append('\n');
+ }
+ }
+ } else {
+ for (int i = 0; i < size; i++) {
+ b.append('[');
+ for (int k = 0; k < projectionSize; k++) {
+ int projIndex = projectedColumns[k];
+ ColumnVector cv = cols[projIndex];
+ if (k > 0) {
+ b.append(", ");
}
+ cv.stringifyValue(b, i);
+ }
+ b.append(']');
+ if (i < size - 1) {
+ b.append('\n');
}
}
- } catch (HiveException ex) {
- throw new RuntimeException(ex);
}
return b.toString();
}
@@ -180,10 +164,6 @@ public class VectorizedRowBatch implements Writable {
throw new UnsupportedOperationException("Don't call me");
}
- public void setValueWriters(VectorExpressionWriter[] valueWriters) {
- this.valueWriters = valueWriters;
- }
-
/**
* Resets the row batch to default state
* - sets selectedInUse to false