You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2013/03/29 01:26:47 UTC
svn commit: r1462363 -
/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
Author: gangtimliu
Date: Fri Mar 29 00:26:46 2013
New Revision: 1462363
URL: http://svn.apache.org/r1462363
Log:
HIVE-4157: ORC runs out of heap when writing (Kevin Wilfong vi Gang Tim Liu)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java?rev=1462363&r1=1462362&r2=1462363&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java Fri Mar 29 00:26:46 2013
@@ -45,17 +45,14 @@ class OutStream extends PositionedOutput
this.bufferSize = bufferSize;
this.codec = codec;
this.receiver = receiver;
- getNewInputBuffer();
}
public void clear() throws IOException {
- current.position(codec == null ? 0 : HEADER_SIZE);
- if (compressed != null) {
- compressed.clear();
- }
- if (overflow != null) {
- overflow.clear();
- }
+ uncompressedBytes = 0;
+ compressedBytes = 0;
+ compressed = null;
+ overflow = null;
+ current = null;
}
/**
@@ -100,6 +97,9 @@ class OutStream extends PositionedOutput
@Override
public void write(int i) throws IOException {
+ if (current == null) {
+ getNewInputBuffer();
+ }
if (current.remaining() < 1) {
spill();
}
@@ -109,6 +109,9 @@ class OutStream extends PositionedOutput
@Override
public void write(byte[] bytes, int offset, int length) throws IOException {
+ if (current == null) {
+ getNewInputBuffer();
+ }
int remaining = Math.min(current.remaining(), length);
current.put(bytes, offset, remaining);
uncompressedBytes += remaining;
@@ -125,7 +128,7 @@ class OutStream extends PositionedOutput
private void spill() throws java.io.IOException {
// if there isn't anything in the current buffer, don't spill
- if (current.position() == (codec == null ? 0 : HEADER_SIZE)) {
+ if (current == null || current.position() == (codec == null ? 0 : HEADER_SIZE)) {
return;
}
flip();
@@ -211,8 +214,7 @@ class OutStream extends PositionedOutput
receiver.output(compressed);
compressed = null;
}
- uncompressedBytes = 0;
- compressedBytes = 0;
+ clear();
}
@Override