You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tg...@apache.org on 2017/03/20 16:26:18 UTC
[1/3] beam git commit: Only advance Index in CreateSource if an
element exists
Repository: beam
Updated Branches:
refs/heads/master 6816d9870 -> 6fa1fca4f
Only advance Index in CreateSource if an element exists
This prevents an IndexOutOfBoundsException when the index in Create
overflows.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/636185ea
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/636185ea
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/636185ea
Branch: refs/heads/master
Commit: 636185ea41c6f676e1eb6fe51d775638c6da895d
Parents: 6816d98
Author: Pei He <pe...@gmail.com>
Authored: Mon Mar 20 08:46:50 2017 +0800
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Mar 20 09:23:57 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/beam/sdk/transforms/Create.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/636185ea/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
index ffc2d8d..fa3b412 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
@@ -464,11 +464,11 @@ public class Create<T> {
@Override
protected boolean advanceImpl() throws IOException {
CreateSource<T> source = getCurrentSource();
- index++;
- if (index >= source.allElementsBytes.size()) {
+ if (index + 1 >= source.allElementsBytes.size()) {
next = null;
return false;
}
+ index++;
next =
Optional.fromNullable(
CoderUtils.decodeFromByteArray(source.coder, source.allElementsBytes.get(index)));
[2/3] beam git commit: Do not call advance when all elements are
consumed
Posted by tg...@apache.org.
Do not call advance when all elements are consumed
This prevents UnboundedReadFromBoundedSource from attempting to read
elements from a reader where elements are known to not exist. This
defends against bounded readers which expect to be discarded the first
time they return false from advance().
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/00f55f8e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/00f55f8e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/00f55f8e
Branch: refs/heads/master
Commit: 00f55f8e3bebf70de1d5497c40e0950b8bd6cbdd
Parents: 636185e
Author: Pei He <pe...@gmail.com>
Authored: Mon Mar 20 08:51:04 2017 +0800
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Mar 20 09:24:46 2017 -0700
----------------------------------------------------------------------
.../beam/runners/core/UnboundedReadFromBoundedSource.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/00f55f8e/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
index 3073076..0c173a0 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
@@ -458,22 +458,28 @@ public class UnboundedReadFromBoundedSource<T> extends PTransform<PBegin, PColle
private PipelineOptions options;
private @Nullable BoundedReader<T> reader;
private boolean closed;
+ private boolean readerDone;
public ResidualSource(BoundedSource<T> residualSource, PipelineOptions options) {
this.residualSource = checkNotNull(residualSource, "residualSource");
this.options = checkNotNull(options, "options");
this.reader = null;
this.closed = false;
+ this.readerDone = false;
}
private boolean advance() throws IOException {
checkArgument(!closed, "advance() call on closed %s", getClass().getName());
+ if (readerDone) {
+ return false;
+ }
if (reader == null) {
reader = residualSource.createReader(options);
- return reader.start();
+ readerDone = !reader.start();
} else {
- return reader.advance();
+ readerDone = !reader.advance();
}
+ return !readerDone;
}
T getCurrent() throws NoSuchElementException {
[3/3] beam git commit: This closes #2262
Posted by tg...@apache.org.
This closes #2262
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/6fa1fca4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/6fa1fca4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/6fa1fca4
Branch: refs/heads/master
Commit: 6fa1fca4f23a36be6e43fd31f93f28990475d805
Parents: 6816d98 00f55f8
Author: Thomas Groh <tg...@google.com>
Authored: Mon Mar 20 09:26:06 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Mar 20 09:26:06 2017 -0700
----------------------------------------------------------------------
.../beam/runners/core/UnboundedReadFromBoundedSource.java | 10 ++++++++--
.../main/java/org/apache/beam/sdk/transforms/Create.java | 4 ++--
2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------