You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ch...@apache.org on 2017/08/18 21:44:43 UTC
[1/2] beam git commit: Updates ByteKeyRangeTracker so that
getFractionConsumed() does not fail for completed trackers.
Repository: beam
Updated Branches:
refs/heads/master d03a1284c -> ae9a2dcfd
Updates ByteKeyRangeTracker so that getFractionConsumed() does not fail for completed trackers.
After this update:
* getFractionConsumed() returns 1.0 after markDone() is set.
* getFractionConsumed() returns 1.0 after tryReturnRecordAt() is invoked for a position that is larger than or equal to the end key.
This is similar to how getFractionConsumed() method of OffsetRangeTracker is implemented.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1b81f1dc
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1b81f1dc
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1b81f1dc
Branch: refs/heads/master
Commit: 1b81f1dc2bfad434fb764c61106679b4d6c94377
Parents: d03a128
Author: chamikara@google.com <ch...@google.com>
Authored: Thu Aug 10 17:35:37 2017 -0700
Committer: chamikara@google.com <ch...@google.com>
Committed: Fri Aug 18 14:09:39 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 5 +++++
.../sdk/io/range/ByteKeyRangeTrackerTest.java | 23 ++++++++++++++++++++
2 files changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/1b81f1dc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
index b889ec7..509e434 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
@@ -127,7 +127,12 @@ public final class ByteKeyRangeTracker implements RangeTracker<ByteKey> {
public synchronized double getFractionConsumed() {
if (position == null) {
return 0;
+ } else if (done) {
+ return 1.0;
+ } else if (position.compareTo(range.getEndKey()) >= 0) {
+ return 1.0;
}
+
return range.estimateFractionForKey(position);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/1b81f1dc/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTrackerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTrackerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTrackerTest.java
index 8deaf44..0523d75 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTrackerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTrackerTest.java
@@ -38,6 +38,7 @@ public class ByteKeyRangeTrackerTest {
private static final ByteKey NEW_MIDDLE_KEY = ByteKey.of(0x24);
private static final ByteKey BEFORE_END_KEY = ByteKey.of(0x33);
private static final ByteKey END_KEY = ByteKey.of(0x34);
+ private static final ByteKey KEY_LARGER_THAN_END = ByteKey.of(0x35);
private static final double INITIAL_RANGE_SIZE = 0x34 - 0x12;
private static final ByteKeyRange INITIAL_RANGE = ByteKeyRange.of(INITIAL_START_KEY, END_KEY);
private static final double NEW_RANGE_SIZE = 0x34 - 0x14;
@@ -98,6 +99,28 @@ public class ByteKeyRangeTrackerTest {
assertEquals(1 - 1 / INITIAL_RANGE_SIZE, tracker.getFractionConsumed(), delta);
}
+ @Test
+ public void testGetFractionConsumedAfterDone() {
+ ByteKeyRangeTracker tracker = ByteKeyRangeTracker.of(INITIAL_RANGE);
+ double delta = 0.00001;
+
+ assertTrue(tracker.tryReturnRecordAt(true, INITIAL_START_KEY));
+ tracker.markDone();
+
+ assertEquals(1.0, tracker.getFractionConsumed(), delta);
+ }
+
+ @Test
+ public void testGetFractionConsumedAfterOutOfRangeClaim() {
+ ByteKeyRangeTracker tracker = ByteKeyRangeTracker.of(INITIAL_RANGE);
+ double delta = 0.00001;
+
+ assertTrue(tracker.tryReturnRecordAt(true, INITIAL_START_KEY));
+ assertTrue(tracker.tryReturnRecordAt(false, KEY_LARGER_THAN_END));
+
+ assertEquals(1.0, tracker.getFractionConsumed(), delta);
+ }
+
/** Tests for {@link ByteKeyRangeTracker#getFractionConsumed()} with updated start key. */
@Test
public void testGetFractionConsumedUpdateStartKey() {
[2/2] beam git commit: This closes #3715
Posted by ch...@apache.org.
This closes #3715
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ae9a2dcf
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ae9a2dcf
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ae9a2dcf
Branch: refs/heads/master
Commit: ae9a2dcfdfa3b3b38274cde0f5779ffec2301955
Parents: d03a128 1b81f1d
Author: chamikara@google.com <ch...@google.com>
Authored: Fri Aug 18 14:44:11 2017 -0700
Committer: chamikara@google.com <ch...@google.com>
Committed: Fri Aug 18 14:44:11 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 5 +++++
.../sdk/io/range/ByteKeyRangeTrackerTest.java | 23 ++++++++++++++++++++
2 files changed, 28 insertions(+)
----------------------------------------------------------------------