You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by rb...@apache.org on 2020/03/19 07:36:22 UTC
[hive] branch master updated: HIVE-23002: Optimise
LazyBinaryUtils.writeVLong (Rajesh Balamohan, reviewed by Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
rbalamohan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e182d9c HIVE-23002: Optimise LazyBinaryUtils.writeVLong (Rajesh Balamohan, reviewed by Ashutosh Chauhan)
e182d9c is described below
commit e182d9ce6c09136d13ee889ef069b202f60052ec
Author: Rajesh Balamohan <rb...@apache.org>
AuthorDate: Thu Mar 19 13:06:06 2020 +0530
HIVE-23002: Optimise LazyBinaryUtils.writeVLong (Rajesh Balamohan, reviewed by Ashutosh Chauhan)
---
.../hive/serde2/lazybinary/LazyBinarySerDe.java | 19 ++++++++++++++++---
.../hive/serde2/lazybinary/LazyBinaryUtils.java | 11 +++++++++++
.../lazybinary/fast/LazyBinarySerializeWrite.java | 11 +++++++++--
3 files changed, 36 insertions(+), 5 deletions(-)
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
index 660080c..a96d298 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
@@ -354,12 +354,13 @@ public class LazyBinarySerDe extends AbstractSerDe {
* @param dec
* @param scratchLongs
* @param scratchBytes
+ * @param scratchLongBytes
*/
public static void writeToByteStream(
RandomAccessOutput byteStream,
HiveDecimal dec,
- long[] scratchLongs, byte[] scratchBytes) {
- LazyBinaryUtils.writeVInt(byteStream, dec.scale());
+ long[] scratchLongs, byte[] scratchBytes, byte[] scratchLongBytes) {
+ LazyBinaryUtils.writeVInt(byteStream, dec.scale(), scratchLongBytes);
// Convert decimal into the scratch buffer without allocating a byte[] each time
// for better performance.
@@ -369,7 +370,7 @@ public class LazyBinarySerDe extends AbstractSerDe {
if (byteLength == 0) {
throw new RuntimeException("Decimal to binary conversion failed");
}
- LazyBinaryUtils.writeVInt(byteStream, byteLength);
+ LazyBinaryUtils.writeVInt(byteStream, byteLength, scratchLongBytes);
byteStream.write(scratchBytes, 0, byteLength);
}
@@ -395,6 +396,18 @@ public class LazyBinarySerDe extends AbstractSerDe {
byteStream.write(scratchBytes, 0, byteLength);
}
+ public static void writeToByteStream(
+ RandomAccessOutput byteStream,
+ HiveDecimalWritable decWritable,
+ long[] scratchLongs, byte[] scratchBytes, byte[] scratchLongBytes) {
+ LazyBinaryUtils.writeVInt(byteStream, decWritable.scale(), scratchLongBytes);
+ int byteLength =
+ decWritable.bigIntegerBytes(
+ scratchLongs, scratchBytes);
+ LazyBinaryUtils.writeVInt(byteStream, byteLength, scratchLongBytes);
+ byteStream.write(scratchBytes, 0, byteLength);
+ }
+
/**
* A recursive function that serialize an object to a byte buffer based on its
* object inspector.
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java
index eb028e3..ed521ee 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java
@@ -343,6 +343,11 @@ public final class LazyBinaryUtils {
writeVLong(byteStream, i);
}
+ public static void writeVInt(RandomAccessOutput byteStream, int i,
+ byte[] scratchBytes) {
+ writeVLong(byteStream, i, scratchBytes);
+ }
+
/**
* Read a zero-compressed encoded long from a byte array.
*
@@ -422,6 +427,12 @@ public final class LazyBinaryUtils {
byteStream.write(vLongBytes, 0, len);
}
+ public static void writeVLong(RandomAccessOutput byteStream, long l,
+ byte[] scratchBytes) {
+ int len = LazyBinaryUtils.writeVLongToByteArray(scratchBytes, l);
+ byteStream.write(scratchBytes, 0, len);
+ }
+
public static void writeDouble(RandomAccessOutput byteStream, double d) {
long v = Double.doubleToLongBits(d);
byteStream.write((byte) (v >> 56));
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java
index ae9111f..5085da0 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java
@@ -72,6 +72,7 @@ public class LazyBinarySerializeWrite implements SerializeWrite {
private byte[] vLongBytes;
private long[] scratchLongs;
private byte[] scratchBuffer;
+ private byte[] scratchLongBytes;
private final Field root;
private Deque<Field> stack = new ArrayDeque<>();
@@ -405,12 +406,15 @@ public class LazyBinarySerializeWrite implements SerializeWrite {
if (scratchLongs == null) {
scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN];
scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES];
+ scratchLongBytes = new byte[LazyBinaryUtils.VLONG_BYTES_LEN];
}
LazyBinarySerDe.writeToByteStream(
output,
dec,
scratchLongs,
- scratchBuffer);
+ scratchBuffer,
+ scratchLongBytes
+ );
finishElement();
}
@@ -420,12 +424,15 @@ public class LazyBinarySerializeWrite implements SerializeWrite {
if (scratchLongs == null) {
scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN];
scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES];
+ scratchLongBytes = new byte[LazyBinaryUtils.VLONG_BYTES_LEN];
}
LazyBinarySerDe.writeToByteStream(
output,
decWritable,
scratchLongs,
- scratchBuffer);
+ scratchBuffer,
+ scratchLongBytes
+ );
finishElement();
}