You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2022/08/26 17:00:05 UTC

[beam] branch master updated: [Go SDK] Stream decode values in single iterations (#22904)

This is an automated email from the ASF dual-hosted git repository.

lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 172bd31d01b [Go SDK] Stream decode values in single iterations (#22904)
172bd31d01b is described below

commit 172bd31d01bcc1e87a5aa1e37379a669c81b46bf
Author: Robert Burke <lo...@users.noreply.github.com>
AuthorDate: Fri Aug 26 09:59:56 2022 -0700

    [Go SDK] Stream decode values in single iterations (#22904)
    
    * [Go SDK] Stream decode values in single iterations
    
    Co-authored-by: lostluck <13...@users.noreply.github.com>
---
 sdks/go/pkg/beam/core/runtime/exec/datasource.go   |  51 +++++-
 .../pkg/beam/core/runtime/exec/datasource_test.go  | 113 +++++++------
 sdks/go/pkg/beam/core/runtime/exec/fn.go           |   3 +
 sdks/go/pkg/beam/core/runtime/exec/fullvalue.go    | 174 +++++++++++++++++++++
 .../pkg/beam/core/runtime/exec/fullvalue_test.go   | 173 +++++++++++++++++++-
 sdks/go/pkg/beam/core/runtime/exec/pardo_test.go   |  49 +++++-
 sdks/go/test/integration/primitives/cogbk.go       |  32 ++++
 sdks/go/test/integration/primitives/cogbk_test.go  |   5 +
 8 files changed, 541 insertions(+), 59 deletions(-)

diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource.go b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
index 95c02f0693a..dc44470228c 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasource.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
@@ -55,6 +55,9 @@ type DataSource struct {
 	su chan SplittableUnit
 
 	mu sync.Mutex
+
+	// Whether the downstream transform only iterates a GBK coder once.
+	singleIterate bool
 }
 
 // InitSplittable initializes the SplittableUnit channel from the output unit,
@@ -75,6 +78,14 @@ func (n *DataSource) ID() UnitID {
 
 // Up initializes this datasource.
 func (n *DataSource) Up(ctx context.Context) error {
+	safeToSingleIterate := true
+	switch n.Out.(type) {
+	case *Expand, *Multiplex:
+		// CoGBK Expands aren't safe, as they may re-iterate the GBK stream.
+		// Multiplexes aren't safe, since they re-iterate the GBK stream by default.
+		safeToSingleIterate = false
+	}
+	n.singleIterate = safeToSingleIterate
 	return nil
 }
 
@@ -165,7 +176,7 @@ func (n *DataSource) Process(ctx context.Context) error {
 
 		var valReStreams []ReStream
 		for _, cv := range cvs {
-			values, err := n.makeReStream(ctx, pe, cv, &bcr)
+			values, err := n.makeReStream(ctx, cv, &bcr, len(cvs) == 1 && n.singleIterate)
 			if err != nil {
 				return err
 			}
@@ -181,13 +192,49 @@ func (n *DataSource) Process(ctx context.Context) error {
 	}
 }
 
-func (n *DataSource) makeReStream(ctx context.Context, key *FullValue, cv ElementDecoder, bcr *byteCountReader) (ReStream, error) {
+func (n *DataSource) makeReStream(ctx context.Context, cv ElementDecoder, bcr *byteCountReader, onlyStream bool) (ReStream, error) {
 	// TODO(lostluck) 2020/02/22: Do we include the chunk size, or just the element sizes?
 	size, err := coder.DecodeInt32(bcr.reader)
 	if err != nil {
 		return nil, errors.Wrap(err, "stream size decoding failed")
 	}
 
+	if onlyStream {
+		// If we know the stream won't be re-iterated,
+		// decode elements on demand instead to reduce memory usage.
+		switch {
+		case size >= 0:
+			return &singleUseReStream{
+				r:    bcr,
+				d:    cv,
+				size: int(size),
+			}, nil
+		case size == -1:
+			return &singleUseMultiChunkReStream{
+				r: bcr,
+				d: cv,
+				open: func(bcr *byteCountReader) (Stream, error) {
+					tokenLen, err := coder.DecodeVarInt(bcr.reader)
+					if err != nil {
+						return nil, err
+					}
+					token, err := ioutilx.ReadN(bcr.reader, (int)(tokenLen))
+					if err != nil {
+						return nil, err
+					}
+					r, err := n.state.OpenIterable(ctx, n.SID, token)
+					if err != nil {
+						return nil, err
+					}
+					// We can't re-use the original bcr, since we may get new iterables,
+					// but we can re-use the count itself.
+					r = &byteCountReader{reader: r, count: bcr.count}
+					return &elementStream{r: r, ec: cv}, nil
+				},
+			}, nil
+		}
+	}
+
 	switch {
 	case size >= 0:
 		// Single chunk streams are fully read in and buffered in memory.
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go b/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go
index 848cf279970..46c00e68ee8 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go
@@ -160,63 +160,74 @@ func TestDataSource_Iterators(t *testing.T) {
 		},
 		// TODO: Test progress.
 	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			out := &IteratorCaptureNode{CaptureNode: CaptureNode{UID: 1}}
-			source := &DataSource{
-				UID:   2,
-				SID:   StreamID{PtransformID: "myPTransform"},
-				Name:  test.name,
-				Coder: test.Coder,
-				Out:   out,
-			}
-			dmr, dmw := io.Pipe()
-
-			// Simulate individual state channels with pipes and a channel.
-			sRc := make(chan io.ReadCloser)
-			swFn := func() io.WriteCloser {
-				sr, sw := io.Pipe()
-				sRc <- sr
-				return sw
-			}
-			go test.driver(source.Coder, dmw, swFn, test.keys, test.vals)
+	for _, singleIterate := range []bool{true, false} {
+		for _, test := range tests {
+			t.Run(test.name, func(t *testing.T) {
+				capture := &IteratorCaptureNode{CaptureNode: CaptureNode{UID: 1}}
+				out := Node(capture)
+				units := []Unit{out}
+				uid := 2
+				if singleIterate {
+					out = &Multiplex{UID: UnitID(uid), Out: []Node{capture}}
+					units = append(units, out)
+					uid++
+				}
+				source := &DataSource{
+					UID:   UnitID(uid),
+					SID:   StreamID{PtransformID: "myPTransform"},
+					Name:  test.name,
+					Coder: test.Coder,
+					Out:   out,
+				}
+				units = append(units, source)
+				dmr, dmw := io.Pipe()
+
+				// Simulate individual state channels with pipes and a channel.
+				sRc := make(chan io.ReadCloser)
+				swFn := func() io.WriteCloser {
+					sr, sw := io.Pipe()
+					sRc <- sr
+					return sw
+				}
+				go test.driver(source.Coder, dmw, swFn, test.keys, test.vals)
 
-			constructAndExecutePlanWithContext(t, []Unit{out, source}, DataContext{
-				Data:  &TestDataManager{R: dmr},
-				State: &TestStateReader{Rc: sRc},
-			})
-			if len(out.CapturedInputs) == 0 {
-				t.Fatal("did not capture source output")
-			}
+				constructAndExecutePlanWithContext(t, units, DataContext{
+					Data:  &TestDataManager{R: dmr},
+					State: &TestStateReader{Rc: sRc},
+				})
+				if len(capture.CapturedInputs) == 0 {
+					t.Fatal("did not capture source output")
+				}
 
-			expectedKeys := makeValues(test.keys...)
-			expectedValues := makeValuesNoWindowOrTime(test.vals...)
-			if got, want := len(out.CapturedInputs), len(expectedKeys); got != want {
-				t.Fatalf("lengths don't match: got %v, want %v", got, want)
-			}
-			var iVals []FullValue
-			for _, i := range out.CapturedInputs {
-				iVals = append(iVals, i.Key)
+				expectedKeys := makeValues(test.keys...)
+				expectedValues := makeValuesNoWindowOrTime(test.vals...)
+				if got, want := len(capture.CapturedInputs), len(expectedKeys); got != want {
+					t.Fatalf("lengths don't match: got %v, want %v", got, want)
+				}
+				var iVals []FullValue
+				for _, i := range capture.CapturedInputs {
+					iVals = append(iVals, i.Key)
 
-				if got, want := i.Values, expectedValues; !equalList(got, want) {
-					t.Errorf("DataSource => key(%v) = %#v, want %#v", i.Key, extractValues(got...), extractValues(want...))
+					if got, want := i.Values, expectedValues; !equalList(got, want) {
+						t.Errorf("DataSource => key(%v) = %#v, want %#v", i.Key, extractValues(got...), extractValues(want...))
+					}
 				}
-			}
 
-			if got, want := iVals, expectedKeys; !equalList(got, want) {
-				t.Errorf("DataSource => %#v, want %#v", extractValues(got...), extractValues(want...))
-			}
+				if got, want := iVals, expectedKeys; !equalList(got, want) {
+					t.Errorf("DataSource => %#v, want %#v", extractValues(got...), extractValues(want...))
+				}
 
-			// We're using integers that encode to 1 byte, so do some quick math to validate.
-			sizeOfSmallInt := 1
-			snap := quickTestSnapshot(source, int64(len(test.keys)))
-			snap.pcol.SizeSum = int64(len(test.keys) * (1 + len(test.vals)) * sizeOfSmallInt)
-			snap.pcol.SizeMin = int64((1 + len(test.vals)) * sizeOfSmallInt)
-			snap.pcol.SizeMax = int64((1 + len(test.vals)) * sizeOfSmallInt)
-			if got, want := source.Progress(), snap; got != want {
-				t.Errorf("progress didn't match: got %v, want %v", got, want)
-			}
-		})
+				// We're using integers that encode to 1 byte, so do some quick math to validate.
+				sizeOfSmallInt := 1
+				snap := quickTestSnapshot(source, int64(len(test.keys)))
+				snap.pcol.SizeSum = int64(len(test.keys) * (1 + len(test.vals)) * sizeOfSmallInt)
+				snap.pcol.SizeMin = int64((1 + len(test.vals)) * sizeOfSmallInt)
+				snap.pcol.SizeMax = int64((1 + len(test.vals)) * sizeOfSmallInt)
+				if got, want := source.Progress(), snap; got != want {
+					t.Errorf("progress didn't match: got %v, want %v", got, want)
+				}
+			})
+		}
 	}
 }
 
diff --git a/sdks/go/pkg/beam/core/runtime/exec/fn.go b/sdks/go/pkg/beam/core/runtime/exec/fn.go
index 80e80c647c7..1f6f196bc28 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/fn.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/fn.go
@@ -235,6 +235,9 @@ func (n *invoker) Invoke(ctx context.Context, pn typex.PaneInfo, ws []typex.Wind
 			it := makeIter(param.T, iter)
 			it.Init()
 			args[in[i]] = it.Value()
+			// Ensure main value iterators are reset & closed after the invoke to avoid
+			// short read problems.
+			defer it.Reset()
 			i++
 		}
 	}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go b/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go
index 3b7babaf02c..e89179ce7d7 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go
@@ -20,9 +20,11 @@ import (
 	"io"
 	"reflect"
 
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/sdf"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors"
 )
 
 // TODO(herohde) 1/29/2018: using FullValue for nested KVs is somewhat of a hack
@@ -189,3 +191,175 @@ func ReadAll(rs ReStream) ([]FullValue, error) {
 		ret = append(ret, *elm)
 	}
 }
+
+// singleUseReStream is a decode on demand ReStream.
+// Can only produce a single Stream because it consumes the reader.
+// Must not be used for streams that might be re-iterated, causing Open
+// to be called twice.
+type singleUseReStream struct {
+	r    io.Reader
+	d    ElementDecoder
+	size int // The number of elements in this stream.
+}
+
+// Open returns the Stream from the start of the in-memory reader. Returns error if called twice.
+func (n *singleUseReStream) Open() (Stream, error) {
+	if n.r == nil {
+		return nil, errors.New("decodeReStream opened twice!")
+	}
+	ret := &decodeStream{r: n.r, d: n.d, size: n.size}
+	n.r = nil
+	n.d = nil
+	return ret, nil
+}
+
+// decodeStream is a decode on demand Stream, that decodes size elements from the provided
+// io.Reader.
+type decodeStream struct {
+	r          io.Reader
+	d          ElementDecoder
+	next, size int
+	ret        FullValue
+}
+
+// Close causes subsequent calls to Read to return io.EOF, and drains the remaining element count
+// from the reader.
+func (s *decodeStream) Close() error {
+	// On close, if next != size, we must iterate through the rest of the decoding
+	// until the reader is drained. Otherwise we corrupt the read for the next element.
+	//
+	// TODO(https://github.com/apache/beam/issues/22901):
+	// Optimize the case where we have length prefixed values
+	// so we can avoid allocating the values in the first place.
+	for s.next < s.size {
+		err := s.d.DecodeTo(s.r, &s.ret)
+		if err != nil {
+			return errors.Wrap(err, "decodeStream value decode failed on close")
+		}
+		s.next++
+	}
+	s.r = nil
+	s.d = nil
+	s.ret = FullValue{}
+	return nil
+}
+
+// Read produces the next value in the stream.
+func (s *decodeStream) Read() (*FullValue, error) {
+	if s.r == nil || s.next == s.size {
+		return nil, io.EOF
+	}
+	err := s.d.DecodeTo(s.r, &s.ret)
+	if err != nil {
+		return nil, errors.Wrap(err, "decodeStream value decode failed")
+	}
+	s.next++
+	return &s.ret, nil
+}
+
+// singleUseMultiChunkReStream is a decode on demand restream, that can handle a multi-chunk streams.
+// Can only produce a single Stream because it consumes the reader.
+// Must not be used for streams that might be re-iterated, causing Open to be called twice.
+type singleUseMultiChunkReStream struct {
+	r *byteCountReader
+	d ElementDecoder
+
+	open func(*byteCountReader) (Stream, error)
+}
+
+// Open returns the Stream from the start of the in-memory ReStream. Returns error if called twice.
+func (n *singleUseMultiChunkReStream) Open() (Stream, error) {
+	if n.r == nil {
+		return nil, errors.New("decodeReStream opened twice!")
+	}
+	ret := &decodeMultiChunkStream{r: n.r, d: n.d, open: n.open}
+	n.r = nil
+	n.d = nil
+	return ret, nil
+}
+
+// decodeMultiChunkStream is a simple decode on demand Stream.
+type decodeMultiChunkStream struct {
+	r           *byteCountReader
+	d           ElementDecoder
+	ret         FullValue
+	next, chunk int64
+
+	open   func(r *byteCountReader) (Stream, error)
+	stream Stream
+}
+
+// Close releases the buffer, closing the stream.
+func (s *decodeMultiChunkStream) Close() error {
+	// On close, if next < size, we must iterate through the rest of the decoding
+	// until the reader is drained. Otherwise we corrupt the read for the next element.
+	// TODO(https://github.com/apache/beam/issues/22901):
+	// Optimize the case where we have length prefixed values
+	// so we can avoid allocating the values in the first place.
+	for s.next < s.chunk {
+		err := s.d.DecodeTo(s.r, &s.ret)
+		if err != nil {
+			return errors.Wrap(err, "decodeStream value decode failed on close")
+		}
+		s.next++
+	}
+	if s.stream != nil {
+		s.stream.Close()
+		s.stream = nil
+	}
+	s.r = nil
+	s.d = nil
+	s.ret = FullValue{}
+	return nil
+}
+
+// Read produces the next value in the stream.
+func (s *decodeMultiChunkStream) Read() (*FullValue, error) {
+	// No Reader? We're done.
+	if s.r == nil {
+		return nil, io.EOF
+	}
+	// If we have a stream already, use that.
+	if s.stream != nil {
+		return s.stream.Read()
+	}
+
+	// If our next value is at the chunk size, then re-set the chunk and size.
+	if s.next == s.chunk {
+		s.chunk = 0
+		s.next = 0
+	}
+
+	// We're at the start of a chunk, see if there's a next chunk.
+	if s.chunk == 0 && s.next == 0 {
+		chunk, err := coder.DecodeVarInt(s.r.reader)
+		if err != nil {
+			return nil, errors.Wrap(err, "decodeMultiChunkStream chunk size decoding failed")
+		}
+		s.chunk = chunk
+	}
+	switch {
+	case s.chunk == 0:
+		// If the chunk is still 0, then we're done.
+		s.r = nil
+		s.d = nil
+		s.ret = FullValue{}
+		return nil, io.EOF
+	case s.chunk > 0:
+		err := s.d.DecodeTo(s.r, &s.ret)
+		if err != nil {
+			return nil, errors.Wrap(err, "decodeStream value decode failed")
+		}
+		s.next++
+		return &s.ret, nil
+	case s.chunk == -1:
+		// State Backed Iterable!
+		stream, err := s.open(s.r)
+		if err != nil {
+			return nil, err
+		}
+		s.stream = stream
+		return s.stream.Read()
+	}
+	return nil, errors.Errorf("decodeMultiChunkStream invalid chunk size: %v", s.chunk)
+}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go b/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go
index d1e62dff1d7..93c7038f6ea 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go
@@ -16,14 +16,15 @@
 package exec
 
 import (
+	"io"
 	"reflect"
 	"testing"
 
-	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
-	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx"
-
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx"
 )
 
 func makeInput(vs ...interface{}) []MainInput {
@@ -242,3 +243,169 @@ func TestConvert(t *testing.T) {
 		})
 	}
 }
+
+func TestDecodeStream(t *testing.T) {
+	c := coder.NewVarInt()
+	e := MakeElementEncoder(c)
+	d := MakeElementDecoder(c)
+
+	const size = 10
+	setup := func() *singleUseReStream {
+		r, w := io.Pipe()
+		// Since the io.Pipe is blocking, run in a goroutine.
+		go func() {
+			for i := int64(0); i < size; i++ {
+				e.Encode(&FullValue{Elm: i}, w)
+			}
+		}()
+		return &singleUseReStream{d: d, r: r, size: int(size)}
+	}
+
+	t.Run("ReadAll", func(t *testing.T) {
+		drs := setup()
+		vals, err := ReadAll(drs)
+		if err != nil {
+			t.Fatalf("unable to ReadAll from decodeStream: %v", err)
+		}
+		if got, want := len(vals), int(size); got != want {
+			t.Errorf("unable to ReadAll from decodeStream, got %v elements, want %v elements", got, want)
+		}
+		var wants []interface{}
+		for i := int64(0); i < size; i++ {
+			wants = append(wants, i)
+		}
+		if got, want := vals, makeValuesNoWindowOrTime(wants...); !equalList(got, want) {
+			t.Errorf("unable to ReadAll from decodeStream got %v, want %v", got, want)
+		}
+	})
+	t.Run("ShortReadAndClose", func(t *testing.T) {
+		const shortSize = 5
+		drs := setup()
+		ds, err := drs.Open()
+		if err != nil {
+			t.Fatalf("unable to create decodeStream: %v", err)
+		}
+		var vals []FullValue
+		for i := 0; i < 5; i++ {
+			fv, err := ds.Read()
+			if err != nil {
+				t.Fatalf("unexpected error on decodeStream.Read: %v", err)
+			}
+			vals = append(vals, *fv)
+		}
+		if got, want := len(vals), int(shortSize); got != want {
+			t.Errorf("unable to ReadAll from decodeStream, got %v elements, want %v elements", got, want)
+		}
+		var wants []interface{}
+		for i := int64(0); i < shortSize; i++ {
+			wants = append(wants, i)
+		}
+		if got, want := vals, makeValuesNoWindowOrTime(wants...); !equalList(got, want) {
+			t.Errorf("unable to short read from decodeStream got %v, want %v", got, want)
+		}
+
+		// Check close Behavior.
+		if err := ds.Close(); err != nil {
+			t.Fatalf("unexpected error on decodeStream.Close: %v", err)
+		}
+
+		if fv, err := ds.Read(); err != io.EOF {
+			t.Errorf("unexpected error on decodeStream.Read after close: %v, %v", fv, err)
+		}
+		// Check that next was iterated to equal size
+		dds := ds.(*decodeStream)
+		if got, want := dds.next, size; got != want {
+			t.Errorf("unexpected configuration after decodeStream.Close: got %v, want %v", got, want)
+		}
+
+		// Check that a 2nd stream will fail:
+		if s, err := drs.Open(); err == nil || s != nil {
+			t.Fatalf("unexpected values for second decodeReStream.Open: %T, %v", s, err)
+		}
+	})
+}
+
+func TestDecodeMultiChunkStream(t *testing.T) {
+	c := coder.NewVarInt()
+	e := MakeElementEncoder(c)
+	d := MakeElementDecoder(c)
+
+	const size = 10
+	setup := func() *singleUseMultiChunkReStream {
+		r, w := io.Pipe()
+		// Since the io.Pipe is blocking, run in a goroutine.
+		go func() {
+			coder.EncodeVarInt(size, w)
+			for i := int64(0); i < size; i++ {
+				e.Encode(&FullValue{Elm: i}, w)
+			}
+			coder.EncodeVarInt(0, w)
+		}()
+		var byteCount int
+		bcr := &byteCountReader{reader: r, count: &byteCount}
+		return &singleUseMultiChunkReStream{d: d, r: bcr}
+	}
+
+	t.Run("ReadAll", func(t *testing.T) {
+		drs := setup()
+		vals, err := ReadAll(drs)
+		if err != nil {
+			t.Fatalf("unable to ReadAll from decodeStream: %v", err)
+		}
+		if got, want := len(vals), int(size); got != want {
+			t.Errorf("unable to ReadAll from decodeStream, got %v elements, want %v elements", got, want)
+		}
+		var wants []interface{}
+		for i := int64(0); i < size; i++ {
+			wants = append(wants, i)
+		}
+		if got, want := vals, makeValuesNoWindowOrTime(wants...); !equalList(got, want) {
+			t.Errorf("unable to ReadAll from decodeStream got %v, want %v", got, want)
+		}
+	})
+	t.Run("ShortReadAndClose", func(t *testing.T) {
+		const shortSize = 5
+		drs := setup()
+		ds, err := drs.Open()
+		if err != nil {
+			t.Fatalf("unable to create decodeStream: %v", err)
+		}
+		var vals []FullValue
+		for i := 0; i < 5; i++ {
+			fv, err := ds.Read()
+			if err != nil {
+				t.Fatalf("unexpected error on decodeStream.Read: %v", err)
+			}
+			vals = append(vals, *fv)
+		}
+		if got, want := len(vals), int(shortSize); got != want {
+			t.Errorf("unable to ReadAll from decodeStream, got %v elements, want %v elements", got, want)
+		}
+		var wants []interface{}
+		for i := int64(0); i < shortSize; i++ {
+			wants = append(wants, i)
+		}
+		if got, want := vals, makeValuesNoWindowOrTime(wants...); !equalList(got, want) {
+			t.Errorf("unable to short read from decodeStream got %v, want %v", got, want)
+		}
+
+		// Check close Behavior.
+		if err := ds.Close(); err != nil {
+			t.Fatalf("unexpected error on decodeStream.Close: %v", err)
+		}
+
+		if fv, err := ds.Read(); err != io.EOF {
+			t.Errorf("unexpected error on decodeStream.Read after close: %v, %v", fv, err)
+		}
+		// Check that next was iterated to equal size
+		dds := ds.(*decodeMultiChunkStream)
+		if got, want := dds.next, int64(size); got != want {
+			t.Errorf("unexpected configuration after decodeStream.Close: got %v, want %v", got, want)
+		}
+
+		// Check that a 2nd stream will fail:
+		if s, err := drs.Open(); err == nil || s != nil {
+			t.Fatalf("unexpected values for second decodeReStream.Open: %T, %v", s, err)
+		}
+	})
+}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go b/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
index a446dea77dc..9e9054675c4 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
@@ -50,9 +50,9 @@ func sumFn(n int, a int, b []int, c func(*int) bool, d func() func(*int) bool, e
 	return n
 }
 
-// TestParDo verifies that the ParDo node works correctly for side inputs and emitters. It uses a special
-// dofn that uses all forms of sideinput.
-func TestParDo(t *testing.T) {
+// TestParDo_IterableSideInputsAndEmitters verifies that the ParDo node works correctly for side inputs and emitters.
+// It uses a special dofn that uses all forms of iterable sideinput.
+func TestParDo_IterableSideInputsAndEmitters(t *testing.T) {
 	fn, err := graph.NewDoFn(sumFn)
 	if err != nil {
 		t.Fatalf("invalid function: %v", err)
@@ -102,6 +102,49 @@ func TestParDo(t *testing.T) {
 	}
 }
 
+func shortFn(n int, iter func(*int) bool, emit func(int)) {
+	var i int
+	iter(&i)
+	emit(i + n)
+}
+
+// TestParDo_ShortGBK verifies that the ParDo node works correctly for short read GBKs.
+func TestParDo_ShortGBK(t *testing.T) {
+	fn, err := graph.NewDoFn(shortFn)
+	if err != nil {
+		t.Fatalf("invalid function: %v", err)
+	}
+
+	g := graph.New()
+	nN := g.NewNode(typex.NewCoGBK(typex.New(reflectx.Int), typex.New(reflectx.Int)), window.DefaultWindowingStrategy(), true)
+
+	edge, err := graph.NewParDo(g, g.Root(), fn, []*graph.Node{nN}, nil, nil)
+	if err != nil {
+		t.Fatalf("invalid pardo: %v", err)
+	}
+
+	short := &CaptureNode{UID: 2}
+	pardo := &ParDo{UID: 3, Fn: edge.DoFn, Inbound: edge.Input, Out: []Node{short}, Side: []SideInputAdapter{}}
+	n := &FixedRoot{UID: 4, Elements: makeKeyedInput(42, 10, 20, 30), Out: pardo}
+
+	p, err := NewPlan("a", []Unit{n, pardo, short})
+	if err != nil {
+		t.Fatalf("failed to construct plan: %v", err)
+	}
+
+	if err := p.Execute(context.Background(), "1", DataContext{}); err != nil {
+		t.Fatalf("execute failed: %v", err)
+	}
+	if err := p.Down(context.Background()); err != nil {
+		t.Fatalf("down failed: %v", err)
+	}
+
+	expected := makeValues(52)
+	if !equalList(short.Elements, expected) {
+		t.Errorf("pardo(sumFn) = %v, want %v", extractValues(short.Elements...), extractValues(expected...))
+	}
+}
+
 func windowObserverFn(w typex.Window, word string) string {
 	if _, ok := w.(window.GlobalWindow); ok {
 		return fmt.Sprintf("%v-%v", word, "global")
diff --git a/sdks/go/test/integration/primitives/cogbk.go b/sdks/go/test/integration/primitives/cogbk.go
index ce863424e06..672f7df7f42 100644
--- a/sdks/go/test/integration/primitives/cogbk.go
+++ b/sdks/go/test/integration/primitives/cogbk.go
@@ -44,6 +44,25 @@ func genC(_ []byte, emit func(string, string)) {
 	emit("d", "delta")
 }
 
+func genD(_ []byte, emit func(string, int)) {
+	emit("a", 1)
+	emit("a", 1)
+	emit("a", 1)
+	emit("b", 4)
+	emit("b", 4)
+	emit("c", 6)
+	emit("c", 6)
+	emit("c", 6)
+	emit("c", 6)
+	emit("c", 6)
+}
+
+func shortFn(_ string, ds func(*int) bool, emit func(int)) {
+	var v int
+	ds(&v)
+	emit(v)
+}
+
 func sum(nums func(*int) bool) int {
 	var ret, i int
 	for nums(&i) {
@@ -101,6 +120,19 @@ func CoGBK() *beam.Pipeline {
 	return p
 }
 
+// GBKShortRead tests GBK with a short read on the iterator.
+func GBKShortRead() *beam.Pipeline {
+	p, s := beam.NewPipelineWithRoot()
+
+	ds := beam.ParDo(s, genD, beam.Impulse(s))
+	grouped := beam.GroupByKey(s, ds)
+	short := beam.ParDo(s, shortFn, grouped)
+
+	passert.Sum(s, short, "shorted", 3, 11)
+
+	return p
+}
+
 // Reshuffle tests Reshuffle.
 func Reshuffle() *beam.Pipeline {
 	p, s := beam.NewPipelineWithRoot()
diff --git a/sdks/go/test/integration/primitives/cogbk_test.go b/sdks/go/test/integration/primitives/cogbk_test.go
index e464e7423f1..b278ea63fbb 100644
--- a/sdks/go/test/integration/primitives/cogbk_test.go
+++ b/sdks/go/test/integration/primitives/cogbk_test.go
@@ -38,3 +38,8 @@ func TestReshuffleKV(t *testing.T) {
 	integration.CheckFilters(t)
 	ptest.RunAndValidate(t, ReshuffleKV())
 }
+
+func TestGBKShortRead(t *testing.T) {
+	integration.CheckFilters(t)
+	ptest.RunAndValidate(t, GBKShortRead())
+}