You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2016/02/23 09:17:03 UTC
[1/2] flink git commit: [hotfix] [runtime] Fix processing unmatched
build-side when all buckets spilled
Repository: flink
Updated Branches:
refs/heads/master 8656ed5ea -> 1c48e3462
[hotfix] [runtime] Fix processing unmatched build-side when all buckets spilled
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/03923c34
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/03923c34
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/03923c34
Branch: refs/heads/master
Commit: 03923c3408250625707af93130b14b7886ef5a8c
Parents: 8656ed5
Author: Greg Hogan <co...@greghogan.com>
Authored: Fri Feb 19 16:08:22 2016 -0500
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Feb 23 01:18:18 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/operators/hash/MutableHashTable.java | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/03923c34/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
index ce4e827..3f00bf9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
@@ -1810,6 +1810,11 @@ public class MutableHashTable<BT, PT> implements MemorySegmentSource {
}
}
+ // if all buckets were spilled out of memory
+ if (this.bucketSegment == null) {
+ return null;
+ }
+
// this segment is done. check if there is another chained bucket
final long forwardPointer = this.bucketSegment.getLong(this.bucketInSegmentOffset + HEADER_FORWARD_OFFSET);
if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
@@ -1847,7 +1852,12 @@ public class MutableHashTable<BT, PT> implements MemorySegmentSource {
this.numInSegment++;
}
}
-
+
+ // if all buckets were spilled out of memory
+ if (this.bucketSegment == null) {
+ return null;
+ }
+
// this segment is done. check if there is another chained bucket
final long forwardPointer = this.bucketSegment.getLong(this.bucketInSegmentOffset + HEADER_FORWARD_OFFSET);
if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
[2/2] flink git commit: [FLINK-3385] [runtime] Fix outer join
skipping unprobed partitions
Posted by fh...@apache.org.
[FLINK-3385] [runtime] Fix outer join skipping unprobed partitions
This closes #1680
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1c48e346
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1c48e346
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1c48e346
Branch: refs/heads/master
Commit: 1c48e3462e9ea16615651fc1984e61d5426fa864
Parents: 03923c3
Author: Greg Hogan <co...@greghogan.com>
Authored: Fri Feb 19 14:59:33 2016 -0500
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Feb 23 01:18:42 2016 +0100
----------------------------------------------------------------------
.../runtime/operators/hash/HashPartition.java | 9 +-
.../operators/hash/MutableHashTable.java | 96 ++++++++++++++------
.../operators/hash/ReOpenableHashPartition.java | 21 +++--
.../runtime/operators/hash/HashTableITCase.java | 52 ++++++++++-
4 files changed, 135 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1c48e346/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java
index 97bef4a..44ee163 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java
@@ -339,11 +339,14 @@ public class HashPartition<BT, PT> extends AbstractPagedInputView implements See
}
/**
+ * @param keepUnprobedSpilledPartitions If true then partitions that were spilled but received no further probe
+ * requests will be retained; used for build-side outer joins.
* @return The number of write-behind buffers reclaimable after this method call.
*
* @throws IOException
*/
- public int finalizeProbePhase(List<MemorySegment> freeMemory, List<HashPartition<BT, PT>> spilledPartitions)
+ public int finalizeProbePhase(List<MemorySegment> freeMemory, List<HashPartition<BT, PT>> spilledPartitions,
+ boolean keepUnprobedSpilledPartitions)
throws IOException
{
if (isInMemory()) {
@@ -363,11 +366,11 @@ public class HashPartition<BT, PT> extends AbstractPagedInputView implements See
this.partitionBuffers = null;
return 0;
}
- else if (this.probeSideRecordCounter == 0) {
+ else if (this.probeSideRecordCounter == 0 && !keepUnprobedSpilledPartitions) {
// partition is empty, no spilled buffers
// return the memory buffer
freeMemory.add(this.probeSideBuffer.getCurrentSegment());
-
+
// delete the spill files
this.probeSideChannel.close();
this.buildSideChannel.deleteChannel();
http://git-wip-us.apache.org/repos/asf/flink/blob/1c48e346/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
index 3f00bf9..7d1fc11 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
@@ -288,12 +288,17 @@ public class MutableHashTable<BT, PT> implements MemorySegmentSource {
* Iterator over the elements from the probe side.
*/
protected ProbeIterator<PT> probeIterator;
-
+
+ /**
+ * The reader for the spilled-file of the build partition that is currently read.
+ */
+ private BlockChannelReader<MemorySegment> currentSpilledBuildSide;
+
/**
* The reader for the spilled-file of the probe partition that is currently read.
*/
private BlockChannelReader<MemorySegment> currentSpilledProbeSide;
-
+
/**
* The channel enumerator that is used while processing the current partition to create
* channels for the spill partitions it requires.
@@ -558,38 +563,40 @@ public class MutableHashTable<BT, PT> implements MemorySegmentSource {
}
protected boolean prepareNextPartition() throws IOException {
-
- this.probeMatchedPhase = true;
- this.unmatchedBuildVisited = false;
-
// finalize and cleanup the partitions of the current table
int buffersAvailable = 0;
for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) {
final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i);
p.setFurtherPatitioning(this.furtherPartitioning);
- buffersAvailable += p.finalizeProbePhase(this.availableMemory, this.partitionsPending);
+ buffersAvailable += p.finalizeProbePhase(this.availableMemory, this.partitionsPending, this.buildSideOuterJoin);
}
-
+
this.partitionsBeingBuilt.clear();
this.writeBehindBuffersAvailable += buffersAvailable;
-
+
releaseTable();
+ if (this.currentSpilledBuildSide != null) {
+ this.currentSpilledBuildSide.closeAndDelete();
+ this.currentSpilledBuildSide = null;
+ }
+
if (this.currentSpilledProbeSide != null) {
this.currentSpilledProbeSide.closeAndDelete();
this.currentSpilledProbeSide = null;
}
- // check if there are pending partitions
- if (!this.partitionsPending.isEmpty()) {
- final HashPartition<BT, PT> p = this.partitionsPending.get(0);
-
- // build the next table
- buildTableFromSpilledPartition(p);
+ if (this.partitionsPending.isEmpty()) {
+ // no more data
+ return false;
+ }
+
+ // there are pending partitions
+ final HashPartition<BT, PT> p = this.partitionsPending.get(0);
- // set the probe side - gather memory segments for reading
- LinkedBlockingQueue<MemorySegment> returnQueue = new LinkedBlockingQueue<MemorySegment>();
- this.currentSpilledProbeSide = this.ioManager.createBlockChannelReader(p.getProbeSideChannel().getChannelID(), returnQueue);
+ if (p.probeSideRecordCounter == 0) {
+ // unprobed spilled partitions are only re-processed for a build-side outer join;
+ // there is no need to create a hash table since there are no probe-side records
List<MemorySegment> memory = new ArrayList<MemorySegment>();
MemorySegment seg1 = getNextBuffer();
@@ -601,24 +608,55 @@ public class MutableHashTable<BT, PT> implements MemorySegmentSource {
}
}
else {
- throw new IllegalStateException("Attempting to begin probing of partition without any memory available");
+ throw new IllegalStateException("Attempting to begin reading spilled partition without any memory available");
}
- ChannelReaderInputViewIterator<PT> probeReader = new ChannelReaderInputViewIterator<PT>(this.currentSpilledProbeSide,
- returnQueue, memory, this.availableMemory, this.probeSideSerializer, p.getProbeSideBlockCount());
- this.probeIterator.set(probeReader);
+ this.currentSpilledBuildSide = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID());
+ final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(currentSpilledBuildSide, memory,
+ p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false);
+ final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView,
+ this.availableMemory, this.buildSideSerializer);
+
+ this.unmatchedBuildIterator = inIter;
- // unregister the pending partition
this.partitionsPending.remove(0);
- this.currentRecursionDepth = p.getRecursionLevel() + 1;
-
- // recursively get the next
- return nextRecord();
+
+ return true;
+ }
+
+ this.probeMatchedPhase = true;
+ this.unmatchedBuildVisited = false;
+
+ // build the next table; memory must be allocated after this call
+ buildTableFromSpilledPartition(p);
+
+ // set the probe side - gather memory segments for reading
+ LinkedBlockingQueue<MemorySegment> returnQueue = new LinkedBlockingQueue<MemorySegment>();
+ this.currentSpilledProbeSide = this.ioManager.createBlockChannelReader(p.getProbeSideChannel().getChannelID(), returnQueue);
+
+ List<MemorySegment> memory = new ArrayList<MemorySegment>();
+ MemorySegment seg1 = getNextBuffer();
+ if (seg1 != null) {
+ memory.add(seg1);
+ MemorySegment seg2 = getNextBuffer();
+ if (seg2 != null) {
+ memory.add(seg2);
+ }
}
else {
- // no more data
- return false;
+ throw new IllegalStateException("Attempting to begin probing of partition without any memory available");
}
+
+ ChannelReaderInputViewIterator<PT> probeReader = new ChannelReaderInputViewIterator<PT>(this.currentSpilledProbeSide,
+ returnQueue, memory, this.availableMemory, this.probeSideSerializer, p.getProbeSideBlockCount());
+ this.probeIterator.set(probeReader);
+
+ // unregister the pending partition
+ this.partitionsPending.remove(0);
+ this.currentRecursionDepth = p.getRecursionLevel() + 1;
+
+ // recursively get the next
+ return nextRecord();
}
public boolean nextRecord() throws IOException {
http://git-wip-us.apache.org/repos/asf/flink/blob/1c48e346/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/ReOpenableHashPartition.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/ReOpenableHashPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/ReOpenableHashPartition.java
index 84868ff..78fca67 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/ReOpenableHashPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/ReOpenableHashPartition.java
@@ -61,7 +61,8 @@ public class ReOpenableHashPartition<BT, PT> extends HashPartition<BT, PT> {
@Override
public int finalizeProbePhase(List<MemorySegment> freeMemory,
- List<HashPartition<BT, PT>> spilledPartitions) throws IOException {
+ List<HashPartition<BT, PT>> spilledPartitions,
+ boolean keepUnprobedSpilledPartitions) throws IOException {
if ( furtherPartitioning || recursionLevel != 0 || isRestored) {
if (isInMemory() && initialBuildSideChannel != null && !isRestored) {
// return the overflow segments
@@ -74,22 +75,22 @@ public class ReOpenableHashPartition<BT, PT> extends HashPartition<BT, PT> {
// we already returned the partitionBuffers via the returnQueue.
return 0;
}
- return super.finalizeProbePhase(freeMemory, spilledPartitions);
+ return super.finalizeProbePhase(freeMemory, spilledPartitions, keepUnprobedSpilledPartitions);
}
- if (!isInMemory() && this.probeSideRecordCounter == 0) {
+ if (isInMemory()) {
+ return 0;
+ } else if (this.probeSideRecordCounter == 0 && !keepUnprobedSpilledPartitions) {
freeMemory.add(this.probeSideBuffer.getCurrentSegment());
// delete the spill files
this.probeSideChannel.close();
this.probeSideChannel.deleteChannel();
return 0;
+ } else {
+ this.probeSideBuffer.close();
+ this.probeSideChannel.close(); // finish pending write requests.
+ spilledPartitions.add(this);
+ return 1;
}
- if (isInMemory()) {
- return 0;
- }
- this.probeSideBuffer.close();
- this.probeSideChannel.close(); // finish pending write requests.
- spilledPartitions.add(this);
- return 1;
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/1c48e346/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTableITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTableITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTableITCase.java
index ed709c0..d9d6a25 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTableITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTableITCase.java
@@ -677,7 +677,57 @@ public class HashTableITCase {
this.memManager.release(join.getFreedMemory());
}
-
+
+ /*
+ * Same test as {@link #testSparseProbeSpilling} but using a build-side outer join
+ * that requires spilled build-side records to be returned and counted.
+ */
+ @Test
+ public void testSparseProbeSpillingWithOuterJoin() throws IOException, MemoryAllocationException
+ {
+ final int NUM_BUILD_KEYS = 1000000;
+ final int NUM_BUILD_VALS = 1;
+ final int NUM_PROBE_KEYS = 20;
+ final int NUM_PROBE_VALS = 1;
+
+ MutableObjectIterator<Record> buildInput = new UniformRecordGenerator(
+ NUM_BUILD_KEYS, NUM_BUILD_VALS, false);
+
+ // allocate the memory for the HashTable
+ List<MemorySegment> memSegments;
+ try {
+ memSegments = this.memManager.allocatePages(MEM_OWNER, 96);
+ }
+ catch (MemoryAllocationException maex) {
+ fail("Memory for the Join could not be provided.");
+ return;
+ }
+
+ final MutableHashTable<Record, Record> join = new MutableHashTable<Record, Record>(
+ this.recordBuildSideAccesssor, this.recordProbeSideAccesssor,
+ this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator,
+ memSegments, ioManager);
+ join.open(buildInput, new UniformRecordGenerator(NUM_PROBE_KEYS, NUM_PROBE_VALS, true), true);
+
+ int expectedNumResults = (Math.max(NUM_PROBE_KEYS, NUM_BUILD_KEYS) * NUM_BUILD_VALS)
+ * NUM_PROBE_VALS;
+
+ final Record recordReuse = new Record();
+ int numRecordsInJoinResult = 0;
+
+ while (join.nextRecord()) {
+ MutableObjectIterator<Record> buildSide = join.getBuildSideIterator();
+ while (buildSide.next(recordReuse) != null) {
+ numRecordsInJoinResult++;
+ }
+ }
+ Assert.assertEquals("Wrong number of records in join result.", expectedNumResults, numRecordsInJoinResult);
+
+ join.close();
+
+ this.memManager.release(join.getFreedMemory());
+ }
+
/*
* This test validates a bug fix against former memory loss in the case where a partition was spilled
* during an insert into the same.