You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2013/05/03 03:09:50 UTC
git commit: TAJO-47: RowFile has the duplicated initialization
problem and unflipped ByteBuffer problem. (jihoon)
Updated Branches:
refs/heads/master 39bb519c8 -> 1db6ca01d
TAJO-47: RowFile has the duplicated initialization problem and unflipped ByteBuffer problem. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/1db6ca01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/1db6ca01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/1db6ca01
Branch: refs/heads/master
Commit: 1db6ca01d42288387725c2cb3ca0fafed450f2f9
Parents: 39bb519
Author: Jihoon Son <ji...@apache.org>
Authored: Fri May 3 10:07:57 2013 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Fri May 3 10:07:57 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../src/test/java/tajo/storage/TestRowFile.java | 2 +
.../src/main/java/tajo/storage/RowFile.java | 24 ++++++++------
3 files changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/1db6ca01/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e2cae30..db43b6d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -32,6 +32,9 @@ Release 0.2.0 - unreleased
BUG FIXES
+ TAJO-47: RowFile has the duplicated initialization problem and unflipped
+ ByteBuffer problem. (jihoon)
+
TAJO-38: Update class comment in TaskAttemptContext from Korean to English
(hsaputra)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/1db6ca01/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java b/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java
index 4e98d8b..cd3058f 100644
--- a/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java
+++ b/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java
@@ -118,6 +118,7 @@ public class TestRowFile {
int tupleCnt = 0;
start = System.currentTimeMillis();
Scanner scanner = new RowFile.RowFileScanner(conf, meta, fragment);
+ scanner.init();
while ((tuple=scanner.next()) != null) {
tupleCnt++;
// System.out.println(tuple.toString());
@@ -137,6 +138,7 @@ public class TestRowFile {
System.out.println("range: " + fileStart + ", " + fileLen);
fragment = new Fragment("test.tbl", dataPath, meta, fileStart, fileLen, null);
scanner = new RowFile.RowFileScanner(conf, meta, fragment);
+ scanner.init();
while ((tuple=scanner.next()) != null) {
if (!idSet.remove(tuple.get(0).asInt())) {
System.out.println("duplicated! " + tuple.get(0).asInt());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/1db6ca01/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java
index 4ee8cec..119fcf4 100644
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java
@@ -79,8 +79,6 @@ public class RowFile {
tupleHeaderSize = nullFlags.size() + (2 * Short.SIZE/8);
this.start = fragment.getStartOffset();
this.end = this.start + fragment.getLength();
-
- init();
}
public void init() throws IOException {
@@ -99,7 +97,6 @@ public class RowFile {
}
bufferStartPos = in.getPos();
fillBuffer();
- fillBuffer(); // due to the bug of FSDataInputStream.read(ByteBuffer)
if (start != 0) {
// TODO: improve
@@ -137,22 +134,29 @@ public class RowFile {
return Arrays.equals(checkSync, sync);
}
- private boolean fillBuffer() throws IOException {
+ private int fillBuffer() throws IOException {
bufferStartPos += buffer.position();
buffer.compact();
+ int remain = buffer.remaining();
int read = in.read(buffer);
- if (read < 0) {
- return false;
+ if (read == -1) {
+ buffer.flip();
+ return read;
} else {
+ int totalRead = read;
+ if (remain > totalRead) {
+ read = in.read(buffer);
+ totalRead += read > 0 ? read : 0;
+ }
buffer.flip();
- return true;
+ return totalRead;
}
}
@Override
public Tuple next() throws IOException {
while (buffer.remaining() < SYNC_SIZE) {
- if (!fillBuffer()) {
+ if (fillBuffer() < 0) {
return null;
}
}
@@ -167,7 +171,7 @@ public class RowFile {
}
while (buffer.remaining() < tupleHeaderSize) {
- if (!fillBuffer()) {
+ if (fillBuffer() < 0) {
return null;
}
}
@@ -182,7 +186,7 @@ public class RowFile {
int tupleSize = buffer.getShort();
while (buffer.remaining() < (tupleSize)) {
- if (!fillBuffer()) {
+ if (fillBuffer() < 0) {
return null;
}
}