You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2022/08/26 17:00:05 UTC
[beam] branch master updated: [Go SDK] Stream decode values in single iterations (#22904)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 172bd31d01b [Go SDK] Stream decode values in single iterations (#22904)
172bd31d01b is described below
commit 172bd31d01bcc1e87a5aa1e37379a669c81b46bf
Author: Robert Burke <lo...@users.noreply.github.com>
AuthorDate: Fri Aug 26 09:59:56 2022 -0700
[Go SDK] Stream decode values in single iterations (#22904)
* [Go SDK] Stream decode values in single iterations
Co-authored-by: lostluck <13...@users.noreply.github.com>
---
sdks/go/pkg/beam/core/runtime/exec/datasource.go | 51 +++++-
.../pkg/beam/core/runtime/exec/datasource_test.go | 113 +++++++------
sdks/go/pkg/beam/core/runtime/exec/fn.go | 3 +
sdks/go/pkg/beam/core/runtime/exec/fullvalue.go | 174 +++++++++++++++++++++
.../pkg/beam/core/runtime/exec/fullvalue_test.go | 173 +++++++++++++++++++-
sdks/go/pkg/beam/core/runtime/exec/pardo_test.go | 49 +++++-
sdks/go/test/integration/primitives/cogbk.go | 32 ++++
sdks/go/test/integration/primitives/cogbk_test.go | 5 +
8 files changed, 541 insertions(+), 59 deletions(-)
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource.go b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
index 95c02f0693a..dc44470228c 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasource.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
@@ -55,6 +55,9 @@ type DataSource struct {
su chan SplittableUnit
mu sync.Mutex
+
+ // Whether the downstream transform only iterates a GBK coder once.
+ singleIterate bool
}
// InitSplittable initializes the SplittableUnit channel from the output unit,
@@ -75,6 +78,14 @@ func (n *DataSource) ID() UnitID {
// Up initializes this datasource.
func (n *DataSource) Up(ctx context.Context) error {
+ safeToSingleIterate := true
+ switch n.Out.(type) {
+ case *Expand, *Multiplex:
+ // CoGBK Expands aren't safe, as they may re-iterate the GBK stream.
+ // Multiplexes aren't safe, since they re-iterate the GBK stream by default.
+ safeToSingleIterate = false
+ }
+ n.singleIterate = safeToSingleIterate
return nil
}
@@ -165,7 +176,7 @@ func (n *DataSource) Process(ctx context.Context) error {
var valReStreams []ReStream
for _, cv := range cvs {
- values, err := n.makeReStream(ctx, pe, cv, &bcr)
+ values, err := n.makeReStream(ctx, cv, &bcr, len(cvs) == 1 && n.singleIterate)
if err != nil {
return err
}
@@ -181,13 +192,49 @@ func (n *DataSource) Process(ctx context.Context) error {
}
}
-func (n *DataSource) makeReStream(ctx context.Context, key *FullValue, cv ElementDecoder, bcr *byteCountReader) (ReStream, error) {
+func (n *DataSource) makeReStream(ctx context.Context, cv ElementDecoder, bcr *byteCountReader, onlyStream bool) (ReStream, error) {
// TODO(lostluck) 2020/02/22: Do we include the chunk size, or just the element sizes?
size, err := coder.DecodeInt32(bcr.reader)
if err != nil {
return nil, errors.Wrap(err, "stream size decoding failed")
}
+ if onlyStream {
+ // If we know the stream won't be re-iterated,
+ // decode elements on demand instead to reduce memory usage.
+ switch {
+ case size >= 0:
+ return &singleUseReStream{
+ r: bcr,
+ d: cv,
+ size: int(size),
+ }, nil
+ case size == -1:
+ return &singleUseMultiChunkReStream{
+ r: bcr,
+ d: cv,
+ open: func(bcr *byteCountReader) (Stream, error) {
+ tokenLen, err := coder.DecodeVarInt(bcr.reader)
+ if err != nil {
+ return nil, err
+ }
+ token, err := ioutilx.ReadN(bcr.reader, (int)(tokenLen))
+ if err != nil {
+ return nil, err
+ }
+ r, err := n.state.OpenIterable(ctx, n.SID, token)
+ if err != nil {
+ return nil, err
+ }
+ // We can't re-use the original bcr, since we may get new iterables,
+ // but we can re-use the count itself.
+ r = &byteCountReader{reader: r, count: bcr.count}
+ return &elementStream{r: r, ec: cv}, nil
+ },
+ }, nil
+ }
+ }
+
switch {
case size >= 0:
// Single chunk streams are fully read in and buffered in memory.
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go b/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go
index 848cf279970..46c00e68ee8 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go
@@ -160,63 +160,74 @@ func TestDataSource_Iterators(t *testing.T) {
},
// TODO: Test progress.
}
- for _, test := range tests {
- t.Run(test.name, func(t *testing.T) {
- out := &IteratorCaptureNode{CaptureNode: CaptureNode{UID: 1}}
- source := &DataSource{
- UID: 2,
- SID: StreamID{PtransformID: "myPTransform"},
- Name: test.name,
- Coder: test.Coder,
- Out: out,
- }
- dmr, dmw := io.Pipe()
-
- // Simulate individual state channels with pipes and a channel.
- sRc := make(chan io.ReadCloser)
- swFn := func() io.WriteCloser {
- sr, sw := io.Pipe()
- sRc <- sr
- return sw
- }
- go test.driver(source.Coder, dmw, swFn, test.keys, test.vals)
+ for _, singleIterate := range []bool{true, false} {
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ capture := &IteratorCaptureNode{CaptureNode: CaptureNode{UID: 1}}
+ out := Node(capture)
+ units := []Unit{out}
+ uid := 2
+ if singleIterate {
+ out = &Multiplex{UID: UnitID(uid), Out: []Node{capture}}
+ units = append(units, out)
+ uid++
+ }
+ source := &DataSource{
+ UID: UnitID(uid),
+ SID: StreamID{PtransformID: "myPTransform"},
+ Name: test.name,
+ Coder: test.Coder,
+ Out: out,
+ }
+ units = append(units, source)
+ dmr, dmw := io.Pipe()
+
+ // Simulate individual state channels with pipes and a channel.
+ sRc := make(chan io.ReadCloser)
+ swFn := func() io.WriteCloser {
+ sr, sw := io.Pipe()
+ sRc <- sr
+ return sw
+ }
+ go test.driver(source.Coder, dmw, swFn, test.keys, test.vals)
- constructAndExecutePlanWithContext(t, []Unit{out, source}, DataContext{
- Data: &TestDataManager{R: dmr},
- State: &TestStateReader{Rc: sRc},
- })
- if len(out.CapturedInputs) == 0 {
- t.Fatal("did not capture source output")
- }
+ constructAndExecutePlanWithContext(t, units, DataContext{
+ Data: &TestDataManager{R: dmr},
+ State: &TestStateReader{Rc: sRc},
+ })
+ if len(capture.CapturedInputs) == 0 {
+ t.Fatal("did not capture source output")
+ }
- expectedKeys := makeValues(test.keys...)
- expectedValues := makeValuesNoWindowOrTime(test.vals...)
- if got, want := len(out.CapturedInputs), len(expectedKeys); got != want {
- t.Fatalf("lengths don't match: got %v, want %v", got, want)
- }
- var iVals []FullValue
- for _, i := range out.CapturedInputs {
- iVals = append(iVals, i.Key)
+ expectedKeys := makeValues(test.keys...)
+ expectedValues := makeValuesNoWindowOrTime(test.vals...)
+ if got, want := len(capture.CapturedInputs), len(expectedKeys); got != want {
+ t.Fatalf("lengths don't match: got %v, want %v", got, want)
+ }
+ var iVals []FullValue
+ for _, i := range capture.CapturedInputs {
+ iVals = append(iVals, i.Key)
- if got, want := i.Values, expectedValues; !equalList(got, want) {
- t.Errorf("DataSource => key(%v) = %#v, want %#v", i.Key, extractValues(got...), extractValues(want...))
+ if got, want := i.Values, expectedValues; !equalList(got, want) {
+ t.Errorf("DataSource => key(%v) = %#v, want %#v", i.Key, extractValues(got...), extractValues(want...))
+ }
}
- }
- if got, want := iVals, expectedKeys; !equalList(got, want) {
- t.Errorf("DataSource => %#v, want %#v", extractValues(got...), extractValues(want...))
- }
+ if got, want := iVals, expectedKeys; !equalList(got, want) {
+ t.Errorf("DataSource => %#v, want %#v", extractValues(got...), extractValues(want...))
+ }
- // We're using integers that encode to 1 byte, so do some quick math to validate.
- sizeOfSmallInt := 1
- snap := quickTestSnapshot(source, int64(len(test.keys)))
- snap.pcol.SizeSum = int64(len(test.keys) * (1 + len(test.vals)) * sizeOfSmallInt)
- snap.pcol.SizeMin = int64((1 + len(test.vals)) * sizeOfSmallInt)
- snap.pcol.SizeMax = int64((1 + len(test.vals)) * sizeOfSmallInt)
- if got, want := source.Progress(), snap; got != want {
- t.Errorf("progress didn't match: got %v, want %v", got, want)
- }
- })
+ // We're using integers that encode to 1 byte, so do some quick math to validate.
+ sizeOfSmallInt := 1
+ snap := quickTestSnapshot(source, int64(len(test.keys)))
+ snap.pcol.SizeSum = int64(len(test.keys) * (1 + len(test.vals)) * sizeOfSmallInt)
+ snap.pcol.SizeMin = int64((1 + len(test.vals)) * sizeOfSmallInt)
+ snap.pcol.SizeMax = int64((1 + len(test.vals)) * sizeOfSmallInt)
+ if got, want := source.Progress(), snap; got != want {
+ t.Errorf("progress didn't match: got %v, want %v", got, want)
+ }
+ })
+ }
}
}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/fn.go b/sdks/go/pkg/beam/core/runtime/exec/fn.go
index 80e80c647c7..1f6f196bc28 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/fn.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/fn.go
@@ -235,6 +235,9 @@ func (n *invoker) Invoke(ctx context.Context, pn typex.PaneInfo, ws []typex.Wind
it := makeIter(param.T, iter)
it.Init()
args[in[i]] = it.Value()
+ // Ensure main value iterators are reset & closed after the invoke to avoid
+ // short read problems.
+ defer it.Reset()
i++
}
}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go b/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go
index 3b7babaf02c..e89179ce7d7 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go
@@ -20,9 +20,11 @@ import (
"io"
"reflect"
+ "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/sdf"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx"
+ "github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors"
)
// TODO(herohde) 1/29/2018: using FullValue for nested KVs is somewhat of a hack
@@ -189,3 +191,175 @@ func ReadAll(rs ReStream) ([]FullValue, error) {
ret = append(ret, *elm)
}
}
+
+// singleUseReStream is a decode on demand ReStream.
+// Can only produce a single Stream because it consumes the reader.
+// Must not be used for streams that might be re-iterated, causing Open
+// to be called twice.
+type singleUseReStream struct {
+ r io.Reader
+ d ElementDecoder
+ size int // The number of elements in this stream.
+}
+
+// Open returns the Stream from the start of the in-memory reader. Returns error if called twice.
+func (n *singleUseReStream) Open() (Stream, error) {
+ if n.r == nil {
+ return nil, errors.New("decodeReStream opened twice!")
+ }
+ ret := &decodeStream{r: n.r, d: n.d, size: n.size}
+ n.r = nil
+ n.d = nil
+ return ret, nil
+}
+
+// decodeStream is a decode on demand Stream, that decodes size elements from the provided
+// io.Reader.
+type decodeStream struct {
+ r io.Reader
+ d ElementDecoder
+ next, size int
+ ret FullValue
+}
+
+// Close causes subsequent calls to Read to return io.EOF, and drains the remaining element count
+// from the reader.
+func (s *decodeStream) Close() error {
+ // On close, if next != size, we must iterate through the rest of the decoding
+ // until the reader is drained. Otherwise we corrupt the read for the next element.
+ //
+ // TODO(https://github.com/apache/beam/issues/22901):
+ // Optimize the case where we have length prefixed values
+ // so we can avoid allocating the values in the first place.
+ for s.next < s.size {
+ err := s.d.DecodeTo(s.r, &s.ret)
+ if err != nil {
+ return errors.Wrap(err, "decodeStream value decode failed on close")
+ }
+ s.next++
+ }
+ s.r = nil
+ s.d = nil
+ s.ret = FullValue{}
+ return nil
+}
+
+// Read produces the next value in the stream.
+func (s *decodeStream) Read() (*FullValue, error) {
+ if s.r == nil || s.next == s.size {
+ return nil, io.EOF
+ }
+ err := s.d.DecodeTo(s.r, &s.ret)
+ if err != nil {
+ return nil, errors.Wrap(err, "decodeStream value decode failed")
+ }
+ s.next++
+ return &s.ret, nil
+}
+
+// singleUseMultiChunkReStream is a decode on demand restream, that can handle a multi-chunk streams.
+// Can only produce a single Stream because it consumes the reader.
+// Must not be used for streams that might be re-iterated, causing Open to be called twice.
+type singleUseMultiChunkReStream struct {
+ r *byteCountReader
+ d ElementDecoder
+
+ open func(*byteCountReader) (Stream, error)
+}
+
+// Open returns the Stream from the start of the in-memory ReStream. Returns error if called twice.
+func (n *singleUseMultiChunkReStream) Open() (Stream, error) {
+ if n.r == nil {
+ return nil, errors.New("decodeReStream opened twice!")
+ }
+ ret := &decodeMultiChunkStream{r: n.r, d: n.d, open: n.open}
+ n.r = nil
+ n.d = nil
+ return ret, nil
+}
+
+// decodeMultiChunkStream is a simple decode on demand Stream.
+type decodeMultiChunkStream struct {
+ r *byteCountReader
+ d ElementDecoder
+ ret FullValue
+ next, chunk int64
+
+ open func(r *byteCountReader) (Stream, error)
+ stream Stream
+}
+
+// Close releases the buffer, closing the stream.
+func (s *decodeMultiChunkStream) Close() error {
+ // On close, if next < size, we must iterate through the rest of the decoding
+ // until the reader is drained. Otherwise we corrupt the read for the next element.
+ // TODO(https://github.com/apache/beam/issues/22901):
+ // Optimize the case where we have length prefixed values
+ // so we can avoid allocating the values in the first place.
+ for s.next < s.chunk {
+ err := s.d.DecodeTo(s.r, &s.ret)
+ if err != nil {
+ return errors.Wrap(err, "decodeStream value decode failed on close")
+ }
+ s.next++
+ }
+ if s.stream != nil {
+ s.stream.Close()
+ s.stream = nil
+ }
+ s.r = nil
+ s.d = nil
+ s.ret = FullValue{}
+ return nil
+}
+
+// Read produces the next value in the stream.
+func (s *decodeMultiChunkStream) Read() (*FullValue, error) {
+ // No Reader? We're done.
+ if s.r == nil {
+ return nil, io.EOF
+ }
+ // If we have a stream already, use that.
+ if s.stream != nil {
+ return s.stream.Read()
+ }
+
+ // If our next value is at the chunk size, then re-set the chunk and size.
+ if s.next == s.chunk {
+ s.chunk = 0
+ s.next = 0
+ }
+
+ // We're at the start of a chunk, see if there's a next chunk.
+ if s.chunk == 0 && s.next == 0 {
+ chunk, err := coder.DecodeVarInt(s.r.reader)
+ if err != nil {
+ return nil, errors.Wrap(err, "decodeMultiChunkStream chunk size decoding failed")
+ }
+ s.chunk = chunk
+ }
+ switch {
+ case s.chunk == 0:
+ // If the chunk is still 0, then we're done.
+ s.r = nil
+ s.d = nil
+ s.ret = FullValue{}
+ return nil, io.EOF
+ case s.chunk > 0:
+ err := s.d.DecodeTo(s.r, &s.ret)
+ if err != nil {
+ return nil, errors.Wrap(err, "decodeStream value decode failed")
+ }
+ s.next++
+ return &s.ret, nil
+ case s.chunk == -1:
+ // State Backed Iterable!
+ stream, err := s.open(s.r)
+ if err != nil {
+ return nil, err
+ }
+ s.stream = stream
+ return s.stream.Read()
+ }
+ return nil, errors.Errorf("decodeMultiChunkStream invalid chunk size: %v", s.chunk)
+}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go b/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go
index d1e62dff1d7..93c7038f6ea 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go
@@ -16,14 +16,15 @@
package exec
import (
+ "io"
"reflect"
"testing"
- "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
- "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx"
-
+ "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
+ "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
+ "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx"
)
func makeInput(vs ...interface{}) []MainInput {
@@ -242,3 +243,169 @@ func TestConvert(t *testing.T) {
})
}
}
+
+func TestDecodeStream(t *testing.T) {
+ c := coder.NewVarInt()
+ e := MakeElementEncoder(c)
+ d := MakeElementDecoder(c)
+
+ const size = 10
+ setup := func() *singleUseReStream {
+ r, w := io.Pipe()
+ // Since the io.Pipe is blocking, run in a goroutine.
+ go func() {
+ for i := int64(0); i < size; i++ {
+ e.Encode(&FullValue{Elm: i}, w)
+ }
+ }()
+ return &singleUseReStream{d: d, r: r, size: int(size)}
+ }
+
+ t.Run("ReadAll", func(t *testing.T) {
+ drs := setup()
+ vals, err := ReadAll(drs)
+ if err != nil {
+ t.Fatalf("unable to ReadAll from decodeStream: %v", err)
+ }
+ if got, want := len(vals), int(size); got != want {
+ t.Errorf("unable to ReadAll from decodeStream, got %v elements, want %v elements", got, want)
+ }
+ var wants []interface{}
+ for i := int64(0); i < size; i++ {
+ wants = append(wants, i)
+ }
+ if got, want := vals, makeValuesNoWindowOrTime(wants...); !equalList(got, want) {
+ t.Errorf("unable to ReadAll from decodeStream got %v, want %v", got, want)
+ }
+ })
+ t.Run("ShortReadAndClose", func(t *testing.T) {
+ const shortSize = 5
+ drs := setup()
+ ds, err := drs.Open()
+ if err != nil {
+ t.Fatalf("unable to create decodeStream: %v", err)
+ }
+ var vals []FullValue
+ for i := 0; i < 5; i++ {
+ fv, err := ds.Read()
+ if err != nil {
+ t.Fatalf("unexpected error on decodeStream.Read: %v", err)
+ }
+ vals = append(vals, *fv)
+ }
+ if got, want := len(vals), int(shortSize); got != want {
+ t.Errorf("unable to ReadAll from decodeStream, got %v elements, want %v elements", got, want)
+ }
+ var wants []interface{}
+ for i := int64(0); i < shortSize; i++ {
+ wants = append(wants, i)
+ }
+ if got, want := vals, makeValuesNoWindowOrTime(wants...); !equalList(got, want) {
+ t.Errorf("unable to short read from decodeStream got %v, want %v", got, want)
+ }
+
+ // Check close Behavior.
+ if err := ds.Close(); err != nil {
+ t.Fatalf("unexpected error on decodeStream.Close: %v", err)
+ }
+
+ if fv, err := ds.Read(); err != io.EOF {
+ t.Errorf("unexpected error on decodeStream.Read after close: %v, %v", fv, err)
+ }
+ // Check that next was iterated to equal size
+ dds := ds.(*decodeStream)
+ if got, want := dds.next, size; got != want {
+ t.Errorf("unexpected configuration after decodeStream.Close: got %v, want %v", got, want)
+ }
+
+ // Check that a 2nd stream will fail:
+ if s, err := drs.Open(); err == nil || s != nil {
+ t.Fatalf("unexpected values for second decodeReStream.Open: %T, %v", s, err)
+ }
+ })
+}
+
+func TestDecodeMultiChunkStream(t *testing.T) {
+ c := coder.NewVarInt()
+ e := MakeElementEncoder(c)
+ d := MakeElementDecoder(c)
+
+ const size = 10
+ setup := func() *singleUseMultiChunkReStream {
+ r, w := io.Pipe()
+ // Since the io.Pipe is blocking, run in a goroutine.
+ go func() {
+ coder.EncodeVarInt(size, w)
+ for i := int64(0); i < size; i++ {
+ e.Encode(&FullValue{Elm: i}, w)
+ }
+ coder.EncodeVarInt(0, w)
+ }()
+ var byteCount int
+ bcr := &byteCountReader{reader: r, count: &byteCount}
+ return &singleUseMultiChunkReStream{d: d, r: bcr}
+ }
+
+ t.Run("ReadAll", func(t *testing.T) {
+ drs := setup()
+ vals, err := ReadAll(drs)
+ if err != nil {
+ t.Fatalf("unable to ReadAll from decodeStream: %v", err)
+ }
+ if got, want := len(vals), int(size); got != want {
+ t.Errorf("unable to ReadAll from decodeStream, got %v elements, want %v elements", got, want)
+ }
+ var wants []interface{}
+ for i := int64(0); i < size; i++ {
+ wants = append(wants, i)
+ }
+ if got, want := vals, makeValuesNoWindowOrTime(wants...); !equalList(got, want) {
+ t.Errorf("unable to ReadAll from decodeStream got %v, want %v", got, want)
+ }
+ })
+ t.Run("ShortReadAndClose", func(t *testing.T) {
+ const shortSize = 5
+ drs := setup()
+ ds, err := drs.Open()
+ if err != nil {
+ t.Fatalf("unable to create decodeStream: %v", err)
+ }
+ var vals []FullValue
+ for i := 0; i < 5; i++ {
+ fv, err := ds.Read()
+ if err != nil {
+ t.Fatalf("unexpected error on decodeStream.Read: %v", err)
+ }
+ vals = append(vals, *fv)
+ }
+ if got, want := len(vals), int(shortSize); got != want {
+ t.Errorf("unable to ReadAll from decodeStream, got %v elements, want %v elements", got, want)
+ }
+ var wants []interface{}
+ for i := int64(0); i < shortSize; i++ {
+ wants = append(wants, i)
+ }
+ if got, want := vals, makeValuesNoWindowOrTime(wants...); !equalList(got, want) {
+ t.Errorf("unable to short read from decodeStream got %v, want %v", got, want)
+ }
+
+ // Check close Behavior.
+ if err := ds.Close(); err != nil {
+ t.Fatalf("unexpected error on decodeStream.Close: %v", err)
+ }
+
+ if fv, err := ds.Read(); err != io.EOF {
+ t.Errorf("unexpected error on decodeStream.Read after close: %v, %v", fv, err)
+ }
+ // Check that next was iterated to equal size
+ dds := ds.(*decodeMultiChunkStream)
+ if got, want := dds.next, int64(size); got != want {
+ t.Errorf("unexpected configuration after decodeStream.Close: got %v, want %v", got, want)
+ }
+
+ // Check that a 2nd stream will fail:
+ if s, err := drs.Open(); err == nil || s != nil {
+ t.Fatalf("unexpected values for second decodeReStream.Open: %T, %v", s, err)
+ }
+ })
+}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go b/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
index a446dea77dc..9e9054675c4 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
@@ -50,9 +50,9 @@ func sumFn(n int, a int, b []int, c func(*int) bool, d func() func(*int) bool, e
return n
}
-// TestParDo verifies that the ParDo node works correctly for side inputs and emitters. It uses a special
-// dofn that uses all forms of sideinput.
-func TestParDo(t *testing.T) {
+// TestParDo_IterableSideInputsAndEmitters verifies that the ParDo node works correctly for side inputs and emitters.
+// It uses a special dofn that uses all forms of iterable sideinput.
+func TestParDo_IterableSideInputsAndEmitters(t *testing.T) {
fn, err := graph.NewDoFn(sumFn)
if err != nil {
t.Fatalf("invalid function: %v", err)
@@ -102,6 +102,49 @@ func TestParDo(t *testing.T) {
}
}
+func shortFn(n int, iter func(*int) bool, emit func(int)) {
+ var i int
+ iter(&i)
+ emit(i + n)
+}
+
+// TestParDo_ShortGBK verifies that the ParDo node works correctly for short read GBKs.
+func TestParDo_ShortGBK(t *testing.T) {
+ fn, err := graph.NewDoFn(shortFn)
+ if err != nil {
+ t.Fatalf("invalid function: %v", err)
+ }
+
+ g := graph.New()
+ nN := g.NewNode(typex.NewCoGBK(typex.New(reflectx.Int), typex.New(reflectx.Int)), window.DefaultWindowingStrategy(), true)
+
+ edge, err := graph.NewParDo(g, g.Root(), fn, []*graph.Node{nN}, nil, nil)
+ if err != nil {
+ t.Fatalf("invalid pardo: %v", err)
+ }
+
+ short := &CaptureNode{UID: 2}
+ pardo := &ParDo{UID: 3, Fn: edge.DoFn, Inbound: edge.Input, Out: []Node{short}, Side: []SideInputAdapter{}}
+ n := &FixedRoot{UID: 4, Elements: makeKeyedInput(42, 10, 20, 30), Out: pardo}
+
+ p, err := NewPlan("a", []Unit{n, pardo, short})
+ if err != nil {
+ t.Fatalf("failed to construct plan: %v", err)
+ }
+
+ if err := p.Execute(context.Background(), "1", DataContext{}); err != nil {
+ t.Fatalf("execute failed: %v", err)
+ }
+ if err := p.Down(context.Background()); err != nil {
+ t.Fatalf("down failed: %v", err)
+ }
+
+ expected := makeValues(52)
+ if !equalList(short.Elements, expected) {
+ t.Errorf("pardo(sumFn) = %v, want %v", extractValues(short.Elements...), extractValues(expected...))
+ }
+}
+
func windowObserverFn(w typex.Window, word string) string {
if _, ok := w.(window.GlobalWindow); ok {
return fmt.Sprintf("%v-%v", word, "global")
diff --git a/sdks/go/test/integration/primitives/cogbk.go b/sdks/go/test/integration/primitives/cogbk.go
index ce863424e06..672f7df7f42 100644
--- a/sdks/go/test/integration/primitives/cogbk.go
+++ b/sdks/go/test/integration/primitives/cogbk.go
@@ -44,6 +44,25 @@ func genC(_ []byte, emit func(string, string)) {
emit("d", "delta")
}
+func genD(_ []byte, emit func(string, int)) {
+ emit("a", 1)
+ emit("a", 1)
+ emit("a", 1)
+ emit("b", 4)
+ emit("b", 4)
+ emit("c", 6)
+ emit("c", 6)
+ emit("c", 6)
+ emit("c", 6)
+ emit("c", 6)
+}
+
+func shortFn(_ string, ds func(*int) bool, emit func(int)) {
+ var v int
+ ds(&v)
+ emit(v)
+}
+
func sum(nums func(*int) bool) int {
var ret, i int
for nums(&i) {
@@ -101,6 +120,19 @@ func CoGBK() *beam.Pipeline {
return p
}
+// GBKShortRead tests GBK with a short read on the iterator.
+func GBKShortRead() *beam.Pipeline {
+ p, s := beam.NewPipelineWithRoot()
+
+ ds := beam.ParDo(s, genD, beam.Impulse(s))
+ grouped := beam.GroupByKey(s, ds)
+ short := beam.ParDo(s, shortFn, grouped)
+
+ passert.Sum(s, short, "shorted", 3, 11)
+
+ return p
+}
+
// Reshuffle tests Reshuffle.
func Reshuffle() *beam.Pipeline {
p, s := beam.NewPipelineWithRoot()
diff --git a/sdks/go/test/integration/primitives/cogbk_test.go b/sdks/go/test/integration/primitives/cogbk_test.go
index e464e7423f1..b278ea63fbb 100644
--- a/sdks/go/test/integration/primitives/cogbk_test.go
+++ b/sdks/go/test/integration/primitives/cogbk_test.go
@@ -38,3 +38,8 @@ func TestReshuffleKV(t *testing.T) {
integration.CheckFilters(t)
ptest.RunAndValidate(t, ReshuffleKV())
}
+
+func TestGBKShortRead(t *testing.T) {
+ integration.CheckFilters(t)
+ ptest.RunAndValidate(t, GBKShortRead())
+}