You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by wa...@apache.org on 2016/12/19 20:48:53 UTC
asterixdb git commit: ASTERIXDB-1751: Spilled partition files will be
cleaned properly.
Repository: asterixdb
Updated Branches:
refs/heads/master d0bd618f2 -> 35ea53b0a
ASTERIXDB-1751: Spilled partition files will be cleaned properly.
- Properly closes the spilled parition file during the build phase of hash join
so that the file will be deleted after the hash join finishes.
Change-Id: Ibdf0baa060efa35a173d00e15f0a348937f2e7f5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1395
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
BAD: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/35ea53b0
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/35ea53b0
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/35ea53b0
Branch: refs/heads/master
Commit: 35ea53b0a597197401d8ea72d53067e792724ce6
Parents: d0bd618
Author: Taewoo Kim <wa...@yahoo.com>
Authored: Sun Dec 18 13:21:05 2016 -0800
Committer: Taewoo Kim <wa...@yahoo.com>
Committed: Mon Dec 19 12:48:21 2016 -0800
----------------------------------------------------------------------
.../dataflow/std/join/OptimizedHybridHashJoin.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/35ea53b0/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
index 0770784..4f85e57 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
@@ -247,10 +247,15 @@ public class OptimizedHybridHashJoin {
break;
}
- for (int pid = spilledStatus.nextSetBit(0); pid >= 0; pid = spilledStatus.nextSetBit(pid + 1)) {
+ for (int pid = spilledStatus.nextSetBit(0); pid >= 0
+ && pid < numOfPartitions; pid = spilledStatus.nextSetBit(pid + 1)) {
if (bufferManager.getNumTuples(pid) > 0) {
bufferManager.flushPartition(pid, getSpillWriterOrCreateNewOneIfNotExist(pid, whichSide));
bufferManager.clearPartition(pid);
+ }
+ // It doesn't matter whether a spilled partition currently holds a tuple in memory or not.
+ // The file that holds the corresponding spilled partition needs to be closed.
+ if (runFileWriters[pid] != null) {
runFileWriters[pid].close();
}
}
@@ -299,7 +304,8 @@ public class OptimizedHybridHashJoin {
}
private int selectPartitionsToReload(int freeSpace, int pid) {
- for (int i = spilledStatus.nextSetBit(pid); i >= 0; i = spilledStatus.nextSetBit(i + 1)) {
+ for (int i = spilledStatus.nextSetBit(pid); i >= 0
+ && i < numOfPartitions; i = spilledStatus.nextSetBit(i + 1)) {
assert buildRFWriters[i].getFileSize() > 0 : "How comes a spilled partition have size 0?";
if (freeSpace >= buildRFWriters[i].getFileSize()) {
return i;