You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tg...@apache.org on 2017/03/20 16:26:18 UTC

[1/3] beam git commit: Only advance Index in CreateSource if an element exists

Repository: beam
Updated Branches:
  refs/heads/master 6816d9870 -> 6fa1fca4f


Only advance Index in CreateSource if an element exists

This prevents an IndexOutOfBoundsException when the index in Create
overflows.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/636185ea
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/636185ea
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/636185ea

Branch: refs/heads/master
Commit: 636185ea41c6f676e1eb6fe51d775638c6da895d
Parents: 6816d98
Author: Pei He <pe...@gmail.com>
Authored: Mon Mar 20 08:46:50 2017 +0800
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Mar 20 09:23:57 2017 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/beam/sdk/transforms/Create.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/636185ea/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
index ffc2d8d..fa3b412 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
@@ -464,11 +464,11 @@ public class Create<T> {
       @Override
       protected boolean advanceImpl() throws IOException {
         CreateSource<T> source = getCurrentSource();
-        index++;
-        if (index >= source.allElementsBytes.size()) {
+        if (index + 1 >= source.allElementsBytes.size()) {
           next = null;
           return false;
         }
+        index++;
         next =
             Optional.fromNullable(
                 CoderUtils.decodeFromByteArray(source.coder, source.allElementsBytes.get(index)));


[2/3] beam git commit: Do not call advance when all elements are consumed

Posted by tg...@apache.org.
Do not call advance when all elements are consumed

This prevents UnboundedReadFromBoundedSource from attempting to read
elements from a reader where elements are known to not exist. This
defends against bounded readers which expect to be discarded the first
time they return false from advance().


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/00f55f8e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/00f55f8e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/00f55f8e

Branch: refs/heads/master
Commit: 00f55f8e3bebf70de1d5497c40e0950b8bd6cbdd
Parents: 636185e
Author: Pei He <pe...@gmail.com>
Authored: Mon Mar 20 08:51:04 2017 +0800
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Mar 20 09:24:46 2017 -0700

----------------------------------------------------------------------
 .../beam/runners/core/UnboundedReadFromBoundedSource.java | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/00f55f8e/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
index 3073076..0c173a0 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
@@ -458,22 +458,28 @@ public class UnboundedReadFromBoundedSource<T> extends PTransform<PBegin, PColle
       private PipelineOptions options;
       private @Nullable BoundedReader<T> reader;
       private boolean closed;
+      private boolean readerDone;
 
       public ResidualSource(BoundedSource<T> residualSource, PipelineOptions options) {
         this.residualSource = checkNotNull(residualSource, "residualSource");
         this.options = checkNotNull(options, "options");
         this.reader = null;
         this.closed = false;
+        this.readerDone = false;
       }
 
       private boolean advance() throws IOException {
         checkArgument(!closed, "advance() call on closed %s", getClass().getName());
+        if (readerDone) {
+          return false;
+        }
         if (reader == null) {
           reader = residualSource.createReader(options);
-          return reader.start();
+          readerDone = !reader.start();
         } else {
-          return reader.advance();
+          readerDone = !reader.advance();
         }
+        return !readerDone;
       }
 
       T getCurrent() throws NoSuchElementException {


[3/3] beam git commit: This closes #2262

Posted by tg...@apache.org.
This closes #2262


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/6fa1fca4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/6fa1fca4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/6fa1fca4

Branch: refs/heads/master
Commit: 6fa1fca4f23a36be6e43fd31f93f28990475d805
Parents: 6816d98 00f55f8
Author: Thomas Groh <tg...@google.com>
Authored: Mon Mar 20 09:26:06 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Mar 20 09:26:06 2017 -0700

----------------------------------------------------------------------
 .../beam/runners/core/UnboundedReadFromBoundedSource.java | 10 ++++++++--
 .../main/java/org/apache/beam/sdk/transforms/Create.java  |  4 ++--
 2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------