You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jk...@apache.org on 2017/06/08 22:28:37 UTC
[1/2] beam git commit: Slight debuggability improvements in BigtableIO
Repository: beam
Updated Branches:
refs/heads/master 911bfbdef -> bc2f97c77
Slight debuggability improvements in BigtableIO
- ByteKeyRangeTracker.splitAtPosition logs the "insane" case first.
- BigtableIO logs the split position at INFO
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/82a6cb61
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/82a6cb61
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/82a6cb61
Branch: refs/heads/master
Commit: 82a6cb6104c0bb179832dabf4433c183743ea983
Parents: 911bfbd
Author: Eugene Kirpichov <ki...@google.com>
Authored: Thu Jun 8 14:51:15 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Jun 8 14:51:15 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 22 ++++++++++++--------
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 2 +-
2 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/82a6cb61/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
index 99717a4..b889ec7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
@@ -71,6 +71,10 @@ public final class ByteKeyRangeTracker implements RangeTracker<ByteKey> {
"Trying to return record which is before the last-returned record");
if (position == null) {
+ LOG.info(
+ "Adjusting range start from {} to {} as position of first returned record",
+ range.getStartKey(),
+ recordStart);
range = range.withStartKey(recordStart);
}
position = recordStart;
@@ -87,6 +91,15 @@ public final class ByteKeyRangeTracker implements RangeTracker<ByteKey> {
@Override
public synchronized boolean trySplitAtPosition(ByteKey splitPosition) {
+ // Sanity check.
+ if (!range.containsKey(splitPosition)) {
+ LOG.warn(
+ "{}: Rejecting split request at {} because it is not within the range.",
+ this,
+ splitPosition);
+ return false;
+ }
+
// Unstarted.
if (position == null) {
LOG.warn(
@@ -106,15 +119,6 @@ public final class ByteKeyRangeTracker implements RangeTracker<ByteKey> {
return false;
}
- // Sanity check.
- if (!range.containsKey(splitPosition)) {
- LOG.warn(
- "{}: Rejecting split request at {} because it is not within the range.",
- this,
- splitPosition);
- return false;
- }
-
range = range.withEndKey(splitPosition);
return true;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/82a6cb61/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 1692cda..62679bb 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -1027,7 +1027,7 @@ public class BigtableIO {
"{}: Failed to interpolate key for fraction {}.", rangeTracker.getRange(), fraction, e);
return null;
}
- LOG.debug(
+ LOG.info(
"Proposing to split {} at fraction {} (key {})", rangeTracker, fraction, splitKey);
BigtableSource primary;
BigtableSource residual;
[2/2] beam git commit: This closes #3331: Slight debuggability
improvements in BigtableIO
Posted by jk...@apache.org.
This closes #3331: Slight debuggability improvements in BigtableIO
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bc2f97c7
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bc2f97c7
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bc2f97c7
Branch: refs/heads/master
Commit: bc2f97c7769bb9b81bc13a181c6e68e5fe219726
Parents: 911bfbd 82a6cb6
Author: Eugene Kirpichov <ki...@google.com>
Authored: Thu Jun 8 14:57:55 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Jun 8 14:57:55 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 22 ++++++++++++--------
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 2 +-
2 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------