You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2023/03/26 22:38:59 UTC

[beam] 01/03: commit with lock

This is an automated email from the ASF dual-hosted git repository.

lostluck pushed a commit to branch datalayerrewrite
in repository https://gitbox.apache.org/repos/asf/beam.git

commit d76c514ba23562a321b9cf283c0d917759f3a16f
Author: lostluck <13...@users.noreply.github.com>
AuthorDate: Sun Mar 26 11:37:42 2023 -0700

    commit with lock
---
 sdks/go/pkg/beam/core/runtime/exec/data.go         |  16 +-
 sdks/go/pkg/beam/core/runtime/exec/datasource.go   | 182 +++++---
 .../pkg/beam/core/runtime/exec/datasource_test.go  | 117 +++--
 .../go/pkg/beam/core/runtime/exec/dynsplit_test.go |  14 +-
 sdks/go/pkg/beam/core/runtime/harness/datamgr.go   | 502 +++++++++++++--------
 .../pkg/beam/core/runtime/harness/datamgr_test.go  | 369 +++++++++++----
 6 files changed, 820 insertions(+), 380 deletions(-)

diff --git a/sdks/go/pkg/beam/core/runtime/exec/data.go b/sdks/go/pkg/beam/core/runtime/exec/data.go
index fdc1e368a52..71954819a74 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/data.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/data.go
@@ -57,10 +57,12 @@ type SideCache interface {
 // DataManager manages external data byte streams. Each data stream can be
 // opened by one consumer only.
 type DataManager interface {
-	// OpenRead opens a closable byte stream for reading.
-	OpenRead(ctx context.Context, id StreamID) (io.ReadCloser, error)
-	// OpenWrite opens a closable byte stream for writing.
+	// OpenElementChan opens a channel for data and timers.
+	OpenElementChan(ctx context.Context, id StreamID, expectedTimerTransforms []string) (<-chan Elements, error)
+	// OpenWrite opens a closable byte stream for data writing.
 	OpenWrite(ctx context.Context, id StreamID) (io.WriteCloser, error)
+	// OpenTimerWrite opens a byte stream for writing timers
+	OpenTimerWrite(ctx context.Context, id StreamID, family string) (io.WriteCloser, error)
 }
 
 // StateReader is the interface for reading side input data.
@@ -91,4 +93,10 @@ type StateReader interface {
 	GetSideInputCache() SideCache
 }
 
-// TODO(herohde) 7/20/2018: user state management
+// Elements holds data or timers sent across the data channel.
+// If TimerFamilyID is populated, it's a timer, otherwise it's
+// data elements.
+type Elements struct {
+	Data, Timers                []byte
+	TimerFamilyID, PtransformID string
+}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource.go b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
index a6347fc8d0e..66c081862f1 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasource.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
@@ -30,6 +30,7 @@ import (
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/ioutilx"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/log"
+	"golang.org/x/exp/maps"
 )
 
 // DataSource is a Root execution unit.
@@ -40,9 +41,12 @@ type DataSource struct {
 	Coder *coder.Coder
 	Out   Node
 	PCol  PCollection // Handles size metrics. Value instead of pointer so it's initialized by default in tests.
+	// OnTimerTransforms maps PtransformIDs to their execution nodes that handle OnTimer callbacks.
+	OnTimerTransforms map[string]*ParDo
 
-	source DataManager
-	state  StateReader
+	source  DataManager
+	state   StateReader
+	curInst string
 
 	index    int64
 	splitIdx int64
@@ -94,20 +98,79 @@ func (n *DataSource) Up(ctx context.Context) error {
 // StartBundle initializes this datasource for the bundle.
 func (n *DataSource) StartBundle(ctx context.Context, id string, data DataContext) error {
 	n.mu.Lock()
+	n.curInst = id
 	n.source = data.Data
 	n.state = data.State
 	n.start = time.Now()
-	n.index = -1
+	n.index = 0
 	n.splitIdx = math.MaxInt64
 	n.mu.Unlock()
 	return n.Out.StartBundle(ctx, id, data)
 }
 
+// splitSuccess is a marker error to indicate we've reached the split index.
+// Akin to io.EOF.
+var splitSuccess = errors.New("split index reached")
+
+// process handles converting elements from the data source to timers.
+//
+// The data and timer callback functions must return an io.EOF if the reader terminates to signal that an additional
+// buffer is desired. On successful splits, [splitSuccess] must be returned to indicate that the
+// PTransform is done processing data for this instruction.
+func (n *DataSource) process(ctx context.Context, data func(bcr *byteCountReader, ptransformID string) error, timer func(bcr *byteCountReader, ptransformID, timerFamilyID string) error) error {
+	// The SID contains this instruction's expected data processing transform (this one).
+	elms, err := n.source.OpenElementChan(ctx, n.SID, maps.Keys(n.OnTimerTransforms))
+	if err != nil {
+		return err
+	}
+
+	n.PCol.resetSize() // initialize the size distribution for this bundle.
+	var r bytes.Reader
+
+	var byteCount int
+	bcr := byteCountReader{reader: &r, count: &byteCount}
+
+	splitPrimaryComplete := map[string]bool{}
+	for {
+		var err error
+		select {
+		case e, ok := <-elms:
+			// Channel closed, so time to exit
+			if !ok {
+				return nil
+			}
+			if splitPrimaryComplete[e.PtransformID] {
+				continue
+			}
+			if len(e.Data) > 0 {
+				r.Reset(e.Data)
+				err = data(&bcr, e.PtransformID)
+			}
+			if len(e.Timers) > 0 {
+				r.Reset(e.Timers)
+				err = timer(&bcr, e.PtransformID, e.TimerFamilyID)
+			}
+
+			if err == splitSuccess {
+				// Returning splitSuccess means we've split, and aren't consuming the remaining buffer.
+				// We mark the PTransform done to ignore further data.
+				splitPrimaryComplete[e.PtransformID] = true
+			} else if err != nil && err != io.EOF {
+				return errors.Wrap(err, "source failed")
+			}
+			// io.EOF means the reader successfully drained.
+			// We're ready for a new buffer.
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}
+
 // ByteCountReader is a passthrough reader that counts all the bytes read through it.
 // It trusts the nested reader to return accurate byte information.
 type byteCountReader struct {
 	count  *int
-	reader io.ReadCloser
+	reader io.Reader
 }
 
 func (r *byteCountReader) Read(p []byte) (int, error) {
@@ -117,7 +180,10 @@ func (r *byteCountReader) Read(p []byte) (int, error) {
 }
 
 func (r *byteCountReader) Close() error {
-	return r.reader.Close()
+	if c, ok := r.reader.(io.Closer); ok {
+		c.Close()
+	}
+	return nil
 }
 
 func (r *byteCountReader) reset() int {
@@ -128,15 +194,6 @@ func (r *byteCountReader) reset() int {
 
 // Process opens the data source, reads and decodes data, kicking off element processing.
 func (n *DataSource) Process(ctx context.Context) ([]*Checkpoint, error) {
-	r, err := n.source.OpenRead(ctx, n.SID)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-	n.PCol.resetSize() // initialize the size distribution for this bundle.
-	var byteCount int
-	bcr := byteCountReader{reader: r, count: &byteCount}
-
 	c := coder.SkipW(n.Coder)
 	wc := MakeWindowDecoder(n.Coder.Window)
 
@@ -155,58 +212,63 @@ func (n *DataSource) Process(ctx context.Context) ([]*Checkpoint, error) {
 	}
 
 	var checkpoints []*Checkpoint
-	for {
-		if n.incrementIndexAndCheckSplit() {
-			break
-		}
-		// TODO(lostluck) 2020/02/22: Should we include window headers or just count the element sizes?
-		ws, t, pn, err := DecodeWindowedValueHeader(wc, r)
-		if err != nil {
-			if err == io.EOF {
-				break
+	err := n.process(ctx, func(bcr *byteCountReader, ptransformID string) error {
+		for {
+			// TODO(lostluck) 2020/02/22: Should we include window headers or just count the element sizes?
+			ws, t, pn, err := DecodeWindowedValueHeader(wc, bcr.reader)
+			if err != nil {
+				return err
 			}
-			return nil, errors.Wrap(err, "source failed")
-		}
-
-		// Decode key or parallel element.
-		pe, err := cp.Decode(&bcr)
-		if err != nil {
-			return nil, errors.Wrap(err, "source decode failed")
-		}
-		pe.Timestamp = t
-		pe.Windows = ws
-		pe.Pane = pn
 
-		var valReStreams []ReStream
-		for _, cv := range cvs {
-			values, err := n.makeReStream(ctx, cv, &bcr, len(cvs) == 1 && n.singleIterate)
+			// Decode key or parallel element.
+			pe, err := cp.Decode(bcr)
 			if err != nil {
-				return nil, err
+				return errors.Wrap(err, "source decode failed")
 			}
-			valReStreams = append(valReStreams, values)
-		}
+			pe.Timestamp = t
+			pe.Windows = ws
+			pe.Pane = pn
 
-		if err := n.Out.ProcessElement(ctx, pe, valReStreams...); err != nil {
-			return nil, err
-		}
-		// Collect the actual size of the element, and reset the bytecounter reader.
-		n.PCol.addSize(int64(bcr.reset()))
-		bcr.reader = r
-
-		// Check if there's a continuation and return residuals
-		// Needs to be done immeadiately after processing to not lose the element.
-		if c := n.getProcessContinuation(); c != nil {
-			cp, err := n.checkpointThis(ctx, c)
-			if err != nil {
-				// Errors during checkpointing should fail a bundle.
-				return nil, err
+			var valReStreams []ReStream
+			for _, cv := range cvs {
+				values, err := n.makeReStream(ctx, cv, bcr, len(cvs) == 1 && n.singleIterate)
+				if err != nil {
+					return err
+				}
+				valReStreams = append(valReStreams, values)
 			}
-			if cp != nil {
-				checkpoints = append(checkpoints, cp)
+
+			if err := n.Out.ProcessElement(ctx, pe, valReStreams...); err != nil {
+				return err
+			}
+			// Collect the actual size of the element, and reset the bytecounter reader.
+			n.PCol.addSize(int64(bcr.reset()))
+
+			// Check if there's a continuation and return residuals
+			// Needs to be done immediately after processing to not lose the element.
+			if c := n.getProcessContinuation(); c != nil {
+				cp, err := n.checkpointThis(ctx, c)
+				if err != nil {
+					// Errors during checkpointing should fail a bundle.
+					return err
+				}
+				if cp != nil {
+					checkpoints = append(checkpoints, cp)
+				}
+			}
+			//	We've finished processing an element, check if we have finished a split.
+			if n.incrementIndexAndCheckSplit() {
+				return splitSuccess
 			}
 		}
-	}
-	return checkpoints, nil
+	},
+		func(bcr *byteCountReader, ptransformID, timerFamilyID string) error {
+			tmap, err := decodeTimer(cp, wc, bcr)
+			log.Infof(ctx, "DEBUGLOG: timer received for: %v and %v - %+v  err: %v", ptransformID, timerFamilyID, tmap, err)
+			return nil
+		})
+
+	return checkpoints, err
 }
 
 func (n *DataSource) makeReStream(ctx context.Context, cv ElementDecoder, bcr *byteCountReader, onlyStream bool) (ReStream, error) {
@@ -313,7 +375,7 @@ func (n *DataSource) makeReStream(ctx context.Context, cv ElementDecoder, bcr *b
 	}
 }
 
-func readStreamToBuffer(cv ElementDecoder, r io.ReadCloser, size int64, buf []FullValue) ([]FullValue, error) {
+func readStreamToBuffer(cv ElementDecoder, r io.Reader, size int64, buf []FullValue) ([]FullValue, error) {
 	for i := int64(0); i < size; i++ {
 		value, err := cv.Decode(r)
 		if err != nil {
@@ -472,7 +534,7 @@ func (n *DataSource) checkpointThis(ctx context.Context, pc sdf.ProcessContinuat
 // The bufSize param specifies the estimated number of elements that will be
 // sent to this DataSource, and is used to be able to perform accurate splits
 // even if the DataSource has not yet received all its elements. A bufSize of
-// 0 or less indicates that its unknown, and so uses the current known size.
+// 0 or less indicates that it's unknown, and so uses the current known size.
 func (n *DataSource) Split(ctx context.Context, splits []int64, frac float64, bufSize int64) (SplitResult, error) {
 	if n == nil {
 		return SplitResult{}, fmt.Errorf("failed to split at requested splits: {%v}, DataSource not initialized", splits)
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go b/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go
index 2da3284f016..ee9614ca0d2 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasource_test.go
@@ -16,7 +16,6 @@
 package exec
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -43,20 +42,20 @@ func TestDataSource_PerElement(t *testing.T) {
 		name     string
 		expected []any
 		Coder    *coder.Coder
-		driver   func(*coder.Coder, io.WriteCloser, []any)
+		driver   func(*coder.Coder, *chanWriter, []any)
 	}{
 		{
 			name:     "perElement",
 			expected: []any{int64(1), int64(2), int64(3), int64(4), int64(5)},
 			Coder:    coder.NewW(coder.NewVarInt(), coder.NewGlobalWindow()),
-			driver: func(c *coder.Coder, pw io.WriteCloser, expected []any) {
+			driver: func(c *coder.Coder, cw *chanWriter, expected []any) {
 				wc := MakeWindowEncoder(c.Window)
 				ec := MakeElementEncoder(coder.SkipW(c))
 				for _, v := range expected {
-					EncodeWindowedValueHeader(wc, window.SingleGlobalWindow, mtime.ZeroTimestamp, typex.NoFiringPane(), pw)
-					ec.Encode(&FullValue{Elm: v}, pw)
+					EncodeWindowedValueHeader(wc, window.SingleGlobalWindow, mtime.ZeroTimestamp, typex.NoFiringPane(), cw)
+					ec.Encode(&FullValue{Elm: v}, cw)
 				}
-				pw.Close()
+				cw.Close()
 			},
 		},
 	}
@@ -70,11 +69,11 @@ func TestDataSource_PerElement(t *testing.T) {
 				Coder: test.Coder,
 				Out:   out,
 			}
-			pr, pw := io.Pipe()
-			go test.driver(source.Coder, pw, test.expected)
+			cw := makeChanWriter()
+			go test.driver(source.Coder, cw, test.expected)
 
 			constructAndExecutePlanWithContext(t, []Unit{out, source}, DataContext{
-				Data: &TestDataManager{R: pr},
+				Data: &TestDataManager{Ch: cw.Ch},
 			})
 
 			validateSource(t, out, source, makeValues(test.expected...))
@@ -98,14 +97,14 @@ func TestDataSource_Iterators(t *testing.T) {
 		name       string
 		keys, vals []any
 		Coder      *coder.Coder
-		driver     func(c *coder.Coder, dmw io.WriteCloser, siwFn func() io.WriteCloser, ks, vs []any)
+		driver     func(c *coder.Coder, dmw *chanWriter, siwFn func() io.WriteCloser, ks, vs []any)
 	}{
 		{
 			name:  "beam:coder:iterable:v1-singleChunk",
 			keys:  []any{int64(42), int64(53)},
 			vals:  []any{int64(1), int64(2), int64(3), int64(4), int64(5)},
 			Coder: coder.NewW(coder.NewCoGBK([]*coder.Coder{coder.NewVarInt(), coder.NewVarInt()}), coder.NewGlobalWindow()),
-			driver: func(c *coder.Coder, dmw io.WriteCloser, _ func() io.WriteCloser, ks, vs []any) {
+			driver: func(c *coder.Coder, dmw *chanWriter, _ func() io.WriteCloser, ks, vs []any) {
 				wc, kc, vc := extractCoders(c)
 				for _, k := range ks {
 					EncodeWindowedValueHeader(wc, window.SingleGlobalWindow, mtime.ZeroTimestamp, typex.NoFiringPane(), dmw)
@@ -123,7 +122,7 @@ func TestDataSource_Iterators(t *testing.T) {
 			keys:  []any{int64(42), int64(53)},
 			vals:  []any{int64(1), int64(2), int64(3), int64(4), int64(5)},
 			Coder: coder.NewW(coder.NewCoGBK([]*coder.Coder{coder.NewVarInt(), coder.NewVarInt()}), coder.NewGlobalWindow()),
-			driver: func(c *coder.Coder, dmw io.WriteCloser, _ func() io.WriteCloser, ks, vs []any) {
+			driver: func(c *coder.Coder, dmw *chanWriter, _ func() io.WriteCloser, ks, vs []any) {
 				wc, kc, vc := extractCoders(c)
 				for _, k := range ks {
 					EncodeWindowedValueHeader(wc, window.SingleGlobalWindow, mtime.ZeroTimestamp, typex.NoFiringPane(), dmw)
@@ -144,7 +143,7 @@ func TestDataSource_Iterators(t *testing.T) {
 			keys:  []any{int64(42), int64(53)},
 			vals:  []any{int64(1), int64(2), int64(3), int64(4), int64(5)},
 			Coder: coder.NewW(coder.NewCoGBK([]*coder.Coder{coder.NewVarInt(), coder.NewVarInt()}), coder.NewGlobalWindow()),
-			driver: func(c *coder.Coder, dmw io.WriteCloser, swFn func() io.WriteCloser, ks, vs []any) {
+			driver: func(c *coder.Coder, dmw *chanWriter, swFn func() io.WriteCloser, ks, vs []any) {
 				wc, kc, vc := extractCoders(c)
 				for _, k := range ks {
 					EncodeWindowedValueHeader(wc, window.SingleGlobalWindow, mtime.ZeroTimestamp, typex.NoFiringPane(), dmw)
@@ -155,6 +154,8 @@ func TestDataSource_Iterators(t *testing.T) {
 					token := []byte(tokenString)
 					coder.EncodeVarInt(int64(len(token)), dmw) // token.
 					dmw.Write(token)
+					dmw.Flush() // Flush here to allow state IO from this goroutine.
+
 					// Each state stream needs to be a different writer, so get a new writer.
 					sw := swFn()
 					for _, v := range vs {
@@ -170,6 +171,7 @@ func TestDataSource_Iterators(t *testing.T) {
 	for _, singleIterate := range []bool{true, false} {
 		for _, test := range tests {
 			t.Run(test.name, func(t *testing.T) {
+				fmt.Println(test.name)
 				capture := &IteratorCaptureNode{CaptureNode: CaptureNode{UID: 1}}
 				out := Node(capture)
 				units := []Unit{out}
@@ -187,8 +189,7 @@ func TestDataSource_Iterators(t *testing.T) {
 					Out:   out,
 				}
 				units = append(units, source)
-				dmr, dmw := io.Pipe()
-
+				cw := makeChanWriter()
 				// Simulate individual state channels with pipes and a channel.
 				sRc := make(chan io.ReadCloser)
 				swFn := func() io.WriteCloser {
@@ -196,10 +197,10 @@ func TestDataSource_Iterators(t *testing.T) {
 					sRc <- sr
 					return sw
 				}
-				go test.driver(source.Coder, dmw, swFn, test.keys, test.vals)
+				go test.driver(source.Coder, cw, swFn, test.keys, test.vals)
 
 				constructAndExecutePlanWithContext(t, units, DataContext{
-					Data:  &TestDataManager{R: dmr},
+					Data:  &TestDataManager{Ch: cw.Ch},
 					State: &TestStateReader{Rc: sRc},
 				})
 				if len(capture.CapturedInputs) == 0 {
@@ -240,7 +241,7 @@ func TestDataSource_Iterators(t *testing.T) {
 
 func TestDataSource_Split(t *testing.T) {
 	elements := []any{int64(1), int64(2), int64(3), int64(4), int64(5)}
-	initSourceTest := func(name string) (*DataSource, *CaptureNode, io.ReadCloser) {
+	initSourceTest := func(name string) (*DataSource, *CaptureNode, chan Elements) {
 		out := &CaptureNode{UID: 1}
 		c := coder.NewW(coder.NewVarInt(), coder.NewGlobalWindow())
 		source := &DataSource{
@@ -250,7 +251,7 @@ func TestDataSource_Split(t *testing.T) {
 			Coder: c,
 			Out:   out,
 		}
-		pr, pw := io.Pipe()
+		cw := makeChanWriter()
 
 		go func(c *coder.Coder, pw io.WriteCloser, elements []any) {
 			wc := MakeWindowEncoder(c.Window)
@@ -260,8 +261,8 @@ func TestDataSource_Split(t *testing.T) {
 				ec.Encode(&FullValue{Elm: v}, pw)
 			}
 			pw.Close()
-		}(c, pw, elements)
-		return source, out, pr
+		}(c, cw, elements)
+		return source, out, cw.Ch
 	}
 
 	tests := []struct {
@@ -289,12 +290,12 @@ func TestDataSource_Split(t *testing.T) {
 			test.expected = elements[:test.splitIdx]
 		}
 		t.Run(test.name, func(t *testing.T) {
-			source, out, pr := initSourceTest(test.name)
+			source, out, ch := initSourceTest(test.name)
 			p, err := NewPlan("a", []Unit{out, source})
 			if err != nil {
 				t.Fatalf("failed to construct plan: %v", err)
 			}
-			dc := DataContext{Data: &TestDataManager{R: pr}}
+			dc := DataContext{Data: &TestDataManager{Ch: ch}}
 			ctx := context.Background()
 
 			// StartBundle resets the source, so no splits can be actuated before then,
@@ -358,7 +359,7 @@ func TestDataSource_Split(t *testing.T) {
 				test.expected = elements[:test.splitIdx]
 			}
 			t.Run(test.name, func(t *testing.T) {
-				source, out, pr := initSourceTest(test.name)
+				source, out, ch := initSourceTest(test.name)
 				unblockCh, blockedCh := make(chan struct{}), make(chan struct{}, 1)
 				// Block on the one less than the desired split,
 				// so the desired split is the first valid split.
@@ -401,7 +402,7 @@ func TestDataSource_Split(t *testing.T) {
 				}()
 
 				constructAndExecutePlanWithContext(t, []Unit{out, blocker, source}, DataContext{
-					Data: &TestDataManager{R: pr},
+					Data: &TestDataManager{Ch: ch},
 				})
 
 				validateSource(t, out, source, makeValues(test.expected...))
@@ -427,12 +428,12 @@ func TestDataSource_Split(t *testing.T) {
 			expected: elements[:3],
 		}
 
-		source, out, pr := initSourceTest("bufSize")
+		source, out, ch := initSourceTest("bufSize")
 		p, err := NewPlan("a", []Unit{out, source})
 		if err != nil {
 			t.Fatalf("failed to construct plan: %v", err)
 		}
-		dc := DataContext{Data: &TestDataManager{R: pr}}
+		dc := DataContext{Data: &TestDataManager{Ch: ch}}
 		ctx := context.Background()
 
 		// StartBundle resets the source, so no splits can be actuated before then,
@@ -490,7 +491,7 @@ func TestDataSource_Split(t *testing.T) {
 			test := test
 			name := fmt.Sprintf("withFraction_%v", test.fraction)
 			t.Run(name, func(t *testing.T) {
-				source, out, pr := initSourceTest(name)
+				source, out, ch := initSourceTest(name)
 				unblockCh, blockedCh := make(chan struct{}), make(chan struct{}, 1)
 				// Block on the one less than the desired split,
 				// so the desired split is the first valid split.
@@ -527,10 +528,10 @@ func TestDataSource_Split(t *testing.T) {
 							t.Errorf("error in Split: got sub-element split = %t, want %t", isSubElm, test.isSubElm)
 						}
 						if isSubElm {
-							if got, want := splitRes.TId, testTransformId; got != want {
+							if got, want := splitRes.TId, testTransformID; got != want {
 								t.Errorf("error in Split: got incorrect Transform Id = %v, want %v", got, want)
 							}
-							if got, want := splitRes.InId, testInputId; got != want {
+							if got, want := splitRes.InId, testInputID; got != want {
 								t.Errorf("error in Split: got incorrect Input Id = %v, want %v", got, want)
 							}
 							if _, ok := splitRes.OW["output1"]; !ok {
@@ -558,7 +559,7 @@ func TestDataSource_Split(t *testing.T) {
 				}()
 
 				constructAndExecutePlanWithContext(t, []Unit{out, blocker, source}, DataContext{
-					Data: &TestDataManager{R: pr},
+					Data: &TestDataManager{Ch: ch},
 				})
 
 				validateSource(t, out, source, makeValues(elements[:test.splitIdx]...))
@@ -571,12 +572,12 @@ func TestDataSource_Split(t *testing.T) {
 
 	// Test expects splitting errors, but for processing to be successful.
 	t.Run("errors", func(t *testing.T) {
-		source, out, pr := initSourceTest("noSplitsUntilStarted")
+		source, out, ch := initSourceTest("noSplitsUntilStarted")
 		p, err := NewPlan("a", []Unit{out, source})
 		if err != nil {
 			t.Fatalf("failed to construct plan: %v", err)
 		}
-		dc := DataContext{Data: &TestDataManager{R: pr}}
+		dc := DataContext{Data: &TestDataManager{Ch: ch}}
 		ctx := context.Background()
 
 		if sr, err := p.Split(ctx, SplitPoints{Splits: []int64{0, 3}, Frac: -1}); err != nil || !sr.Unsuccessful {
@@ -620,8 +621,8 @@ func TestDataSource_Split(t *testing.T) {
 	})
 }
 
-const testTransformId = "transform_id"
-const testInputId = "input_id"
+const testTransformID = "transform_id"
+const testInputID = "input_id"
 
 // TestSplittableUnit is an implementation of the SplittableUnit interface
 // for DataSource tests.
@@ -651,12 +652,12 @@ func (n *TestSplittableUnit) GetProgress() float64 {
 
 // GetTransformId returns a constant transform ID that can be tested for.
 func (n *TestSplittableUnit) GetTransformId() string {
-	return testTransformId
+	return testTransformID
 }
 
 // GetInputId returns a constant input ID that can be tested for.
 func (n *TestSplittableUnit) GetInputId() string {
-	return testInputId
+	return testInputID
 }
 
 // GetOutputWatermark gets the current output watermark of the splittable unit
@@ -966,20 +967,21 @@ func TestCheckpointing(t *testing.T) {
 		}
 
 		enc := MakeElementEncoder(wvERSCoder)
-		var buf bytes.Buffer
+		cw := makeChanWriter()
 
 		// We encode the element several times to ensure we don't
 		// drop any residuals, the root of issue #24931.
 		wantCount := 3
 		for i := 0; i < wantCount; i++ {
-			if err := enc.Encode(value, &buf); err != nil {
+			if err := enc.Encode(value, cw); err != nil {
 				t.Fatalf("couldn't encode value: %v", err)
 			}
 		}
+		cw.Close()
 
 		if err := root.StartBundle(ctx, "testBund", DataContext{
 			Data: &TestDataManager{
-				R: io.NopCloser(&buf),
+				Ch: cw.Ch,
 			},
 		},
 		); err != nil {
@@ -1017,17 +1019,44 @@ func runOnRoots(ctx context.Context, t *testing.T, p *Plan, name string, mthd fu
 }
 
 type TestDataManager struct {
-	R io.ReadCloser
+	Ch chan Elements
 }
 
-func (dm *TestDataManager) OpenRead(ctx context.Context, id StreamID) (io.ReadCloser, error) {
-	return dm.R, nil
+func (dm *TestDataManager) OpenElementChan(ctx context.Context, id StreamID, expectedTimerTransforms []string) (<-chan Elements, error) {
+	return dm.Ch, nil
 }
 
 func (dm *TestDataManager) OpenWrite(ctx context.Context, id StreamID) (io.WriteCloser, error) {
 	return nil, nil
 }
 
+func (dm *TestDataManager) OpenTimerWrite(ctx context.Context, id StreamID, family string) (io.WriteCloser, error) {
+	return nil, nil
+}
+
+type chanWriter struct {
+	Ch  chan Elements
+	Buf []byte
+}
+
+func (cw *chanWriter) Write(p []byte) (int, error) {
+	cw.Buf = append(cw.Buf, p...)
+	return len(p), nil
+}
+
+func (cw *chanWriter) Close() error {
+	cw.Flush()
+	close(cw.Ch)
+	return nil
+}
+
+func (cw *chanWriter) Flush() {
+	cw.Ch <- Elements{Data: cw.Buf, PtransformID: "myPTransform"}
+	cw.Buf = nil
+}
+
+func makeChanWriter() *chanWriter { return &chanWriter{Ch: make(chan Elements, 20)} }
+
 // TestSideInputReader simulates state reads using channels.
 type TestStateReader struct {
 	StateReader
@@ -1079,4 +1108,4 @@ func validateSource(t *testing.T, out *CaptureNode, source *DataSource, expected
 	if !equalList(out.Elements, expected) {
 		t.Errorf("DataSource => %#v, want %#v", extractValues(out.Elements...), extractValues(expected...))
 	}
-}
+}
\ No newline at end of file
diff --git a/sdks/go/pkg/beam/core/runtime/exec/dynsplit_test.go b/sdks/go/pkg/beam/core/runtime/exec/dynsplit_test.go
index 1fa3ae94c86..84c84a8d316 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/dynsplit_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/dynsplit_test.go
@@ -75,10 +75,10 @@ func TestDynamicSplit(t *testing.T) {
 			plan, out := createSdfPlan(t, t.Name(), dfn, cdr)
 
 			// Create thread to send element to pipeline.
-			pr, pw := io.Pipe()
+			cw := makeChanWriter()
 			elm := createElm()
-			go writeElm(elm, cdr, pw)
-			dc := DataContext{Data: &TestDataManager{R: pr}}
+			go writeElm(elm, cdr, cw)
+			dc := DataContext{Data: &TestDataManager{Ch: cw.Ch}}
 
 			// Call driver to coordinate processing & splitting threads.
 			splitRes, procRes := test.driver(context.Background(), plan, dc, sdf)
@@ -92,7 +92,7 @@ func TestDynamicSplit(t *testing.T) {
 				RI:   1,
 				PS:   nil,
 				RS:   nil,
-				TId:  testTransformId,
+				TId:  testTransformID,
 				InId: indexToInputId(0),
 			}
 			if diff := cmp.Diff(splitRes.split, wantSplit, cmpopts.IgnoreFields(SplitResult{}, "PS", "RS")); diff != "" {
@@ -263,7 +263,7 @@ func createSplitTestInCoder() *coder.Coder {
 func createSdfPlan(t *testing.T, name string, fn *graph.DoFn, cdr *coder.Coder) (*Plan, *CaptureNode) {
 	out := &CaptureNode{UID: 0}
 	n := &ParDo{UID: 1, Fn: fn, Out: []Node{out}}
-	sdf := &ProcessSizedElementsAndRestrictions{PDo: n, TfId: testTransformId}
+	sdf := &ProcessSizedElementsAndRestrictions{PDo: n, TfId: testTransformID}
 	ds := &DataSource{
 		UID:   2,
 		SID:   StreamID{PtransformID: "DataSource"},
@@ -281,8 +281,8 @@ func createSdfPlan(t *testing.T, name string, fn *graph.DoFn, cdr *coder.Coder)
 }
 
 // writeElm is meant to be the goroutine for feeding an element to the
-// DataSourc of the test pipeline.
-func writeElm(elm *FullValue, cdr *coder.Coder, pw *io.PipeWriter) {
+// DataSource of the test pipeline.
+func writeElm(elm *FullValue, cdr *coder.Coder, pw io.WriteCloser) {
 	wc := MakeWindowEncoder(cdr.Window)
 	ec := MakeElementEncoder(coder.SkipW(cdr))
 	if err := EncodeWindowedValueHeader(wc, window.SingleGlobalWindow, mtime.ZeroTimestamp, typex.NoFiringPane(), pw); err != nil {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
index 5a9b536b288..0c9c2488edb 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
@@ -17,6 +17,7 @@ package harness
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"sync"
 	"time"
@@ -35,8 +36,9 @@ const (
 // ScopedDataManager scopes the global gRPC data manager to a single instruction.
 // The indirection makes it easier to control access.
 type ScopedDataManager struct {
-	mgr    *DataChannelManager
-	instID instructionID
+	mgr       *DataChannelManager
+	instID    instructionID
+	openPorts []exec.Port
 
 	closed bool
 	mu     sync.Mutex
@@ -47,22 +49,31 @@ func NewScopedDataManager(mgr *DataChannelManager, instID instructionID) *Scoped
 	return &ScopedDataManager{mgr: mgr, instID: instID}
 }
 
-// OpenRead opens an io.ReadCloser on the given stream.
-func (s *ScopedDataManager) OpenRead(ctx context.Context, id exec.StreamID) (io.ReadCloser, error) {
+// OpenWrite opens an io.WriteCloser on the given stream.
+func (s *ScopedDataManager) OpenWrite(ctx context.Context, id exec.StreamID) (io.WriteCloser, error) {
 	ch, err := s.open(ctx, id.Port)
 	if err != nil {
 		return nil, err
 	}
-	return ch.OpenRead(ctx, id.PtransformID, s.instID), nil
+	return ch.OpenWrite(ctx, id.PtransformID, s.instID), nil
 }
 
-// OpenWrite opens an io.WriteCloser on the given stream.
-func (s *ScopedDataManager) OpenWrite(ctx context.Context, id exec.StreamID) (io.WriteCloser, error) {
+// OpenElementChan returns a channel of exec.Elements on the given stream.
+func (s *ScopedDataManager) OpenElementChan(ctx context.Context, id exec.StreamID, expectedTimerTransforms []string) (<-chan exec.Elements, error) {
 	ch, err := s.open(ctx, id.Port)
 	if err != nil {
 		return nil, err
 	}
-	return ch.OpenWrite(ctx, id.PtransformID, s.instID), nil
+	return ch.OpenElementChan(ctx, id.PtransformID, s.instID, expectedTimerTransforms)
+}
+
+// OpenTimerWrite opens an io.WriteCloser on the given stream to write timers
+func (s *ScopedDataManager) OpenTimerWrite(ctx context.Context, id exec.StreamID, family string) (io.WriteCloser, error) {
+	ch, err := s.open(ctx, id.Port)
+	if err != nil {
+		return nil, err
+	}
+	return ch.OpenTimerWrite(ctx, id.PtransformID, s.instID, family), nil
 }
 
 func (s *ScopedDataManager) open(ctx context.Context, port exec.Port) (*DataChannel, error) {
@@ -71,6 +82,7 @@ func (s *ScopedDataManager) open(ctx context.Context, port exec.Port) (*DataChan
 		s.mu.Unlock()
 		return nil, errors.Errorf("instruction %v no longer processing", s.instID)
 	}
+	s.openPorts = append(s.openPorts, port)
 	local := s.mgr
 	s.mu.Unlock()
 
@@ -82,9 +94,9 @@ func (s *ScopedDataManager) Close() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	s.closed = true
-	s.mgr.closeInstruction(s.instID)
+	err := s.mgr.closeInstruction(s.instID, s.openPorts)
 	s.mgr = nil
-	return nil
+	return err
 }
 
 // DataChannelManager manages data channels over the Data API. A fixed number of channels
@@ -124,28 +136,37 @@ func (m *DataChannelManager) Open(ctx context.Context, port exec.Port) (*DataCha
 	return ch, nil
 }
 
-func (m *DataChannelManager) closeInstruction(instID instructionID) {
+func (m *DataChannelManager) closeInstruction(instID instructionID, ports []exec.Port) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	for _, ch := range m.ports {
-		ch.removeInstruction(instID)
+	var firstNonNilError error
+	for _, port := range ports {
+		ch, ok := m.ports[port.URL]
+		if !ok {
+			continue
+		}
+		err := ch.removeInstruction(instID)
+		if err != nil && firstNonNilError == nil {
+			firstNonNilError = err
+		}
 	}
+	return firstNonNilError
 }
 
 // clientID identifies a client of a connected channel.
 type clientID struct {
-	ptransformID string
 	instID       instructionID
+	ptransformID string
 }
 
 // This is a reduced version of the full gRPC interface to help with testing.
-// TODO(wcn): need a compile-time assertion to make sure this stays synced with what's
-// in fnpb.BeamFnData_DataClient
 type dataClient interface {
 	Send(*fnpb.Elements) error
 	Recv() (*fnpb.Elements, error)
 }
 
+var _ dataClient = (fnpb.BeamFnData_DataClient)(nil) // Assert our interfaces are compatible.
+
 // DataChannel manages a single gRPC stream over the Data API. Data from
 // multiple bundles can be multiplexed over this stream. Data is pushed
 // over the channel, so data for a reader may arrive before the reader
@@ -155,8 +176,9 @@ type DataChannel struct {
 	id     string
 	client dataClient
 
-	writers map[instructionID]map[string]*dataWriter
-	readers map[instructionID]map[string]*dataReader
+	writers      map[instructionID]map[string]*dataWriter // PTransformID
+	timerWriters map[instructionID]map[timerKey]*timerWriter
+	channels     map[instructionID]*elementsChan
 
 	// recently terminated instructions
 	endedInstructions map[instructionID]struct{}
@@ -172,6 +194,56 @@ type DataChannel struct {
 	mu sync.Mutex // guards mutable internal data, notably the maps and readErr.
 }
 
+type timerKey struct {
+	ptransformID, family string
+}
+
+// elementsChan abstracts the management for this instruction's channel.
+//
+// The only runner signal that all data for an instruction has been received
+// is when the ch channel has been closed. However, we may receive all data
+// before the instruction begins consuming it, and there may be multiple PTransforms
+// in the instruction that may need data through this channel. Until the instruction
+// arrives, received data needs to be cached, and we cannot close the channel.
+//
+// The channel may only close if the want == got and want > 0.
+// want is set once when the Source requests it.
+// got is incremented only if we receive an IsLast signal for a given
+// instruction/transform pair.
+type elementsChan struct {
+	instID    instructionID
+	mu        sync.Mutex
+	want, got int32
+
+	ch        chan exec.Elements // closed only by read loop
+	closeOnce sync.Once
+
+	done chan struct{} // Forces escape from a blocked write to allow channel close.
+}
+
+// InstructionEnded signals the read loop to close the channel.
+func (ec *elementsChan) InstructionEnded() {
+	close(ec.done)
+}
+
+// Closed indicates if all expected streams are complete
+func (ec *elementsChan) Closed() bool {
+	ec.mu.Lock()
+	defer ec.mu.Unlock()
+	return ec.want > 0 && ec.want == ec.got
+}
+
+// PTransformDone signals that a PTransform has no more data coming to it.
+// If permitted, PTransformDone closes the channel.
+func (ec *elementsChan) PTransformDone() {
+	ec.mu.Lock()
+	defer ec.mu.Unlock()
+	ec.got++
+	if ec.want > 0 && ec.want == ec.got {
+		close(ec.ch)
+	}
+}
+
 func newDataChannel(ctx context.Context, port exec.Port) (*DataChannel, error) {
 	ctx, cancelFn := context.WithCancel(ctx)
 	cc, err := dial(ctx, port.URL, "data", 15*time.Second)
@@ -196,7 +268,8 @@ func makeDataChannel(ctx context.Context, id string, client dataClient, cancelFn
 		id:                id,
 		client:            client,
 		writers:           make(map[instructionID]map[string]*dataWriter),
-		readers:           make(map[instructionID]map[string]*dataReader),
+		timerWriters:      make(map[instructionID]map[timerKey]*timerWriter),
+		channels:          make(map[instructionID]*elementsChan),
 		endedInstructions: make(map[instructionID]struct{}),
 		cancelFn:          cancelFn,
 	}
@@ -214,25 +287,70 @@ func (c *DataChannel) terminateStreamOnError(err error) {
 	}
 }
 
-// OpenRead returns an io.ReadCloser of the data elements for the given instruction and ptransform.
-func (c *DataChannel) OpenRead(ctx context.Context, ptransformID string, instID instructionID) io.ReadCloser {
+// OpenWrite returns an io.WriteCloser of the data elements for the given instruction and ptransform.
+func (c *DataChannel) OpenWrite(ctx context.Context, ptransformID string, instID instructionID) io.WriteCloser {
+	return c.makeWriter(ctx, clientID{ptransformID: ptransformID, instID: instID})
+}
+
+// OpenElementChan returns a channel of typex.Elements for the given instruction and ptransform.
+func (c *DataChannel) OpenElementChan(ctx context.Context, ptransformID string, instID instructionID, expectedTimerTransforms []string) (<-chan exec.Elements, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	cid := clientID{ptransformID: ptransformID, instID: instID}
 	if c.readErr != nil {
-		log.Errorf(ctx, "opening a reader %v on a closed channel", cid)
-		return &errReader{c.readErr}
+		return nil, fmt.Errorf("opening a reader %v on a closed channel. Original error: %w", cid, c.readErr)
 	}
-	return c.makeReader(ctx, cid)
+	//Infof(ctx, "OpenElementChan %v %v", cid, expectedTimerTransforms)
+	return c.makeChannel(ctx, true, cid, expectedTimerTransforms...).ch, nil
 }
 
-// OpenWrite returns an io.WriteCloser of the data elements for the given instruction and ptransform.
-func (c *DataChannel) OpenWrite(ctx context.Context, ptransformID string, instID instructionID) io.WriteCloser {
-	return c.makeWriter(ctx, clientID{ptransformID: ptransformID, instID: instID})
+// makeChannel creates a channel of exec.Elements. It expects to be called while c.mu is held.
+func (c *DataChannel) makeChannel(ctx context.Context, fromSource bool, id clientID, additionalTransforms ...string) *elementsChan {
+	if ec, ok := c.channels[id.instID]; ok {
+		ec.mu.Lock()
+		defer ec.mu.Unlock()
+		if fromSource {
+			ec.want = (1 + int32(len(additionalTransforms)))
+		}
+		if _, ok := c.endedInstructions[id.instID]; ok || (ec.want > 0 && ec.want == ec.got) {
+			ec.got = ec.want
+			close(ec.ch)
+		}
+		return ec
+	}
+
+	ec := &elementsChan{
+		instID: id.instID,
+		ch:     make(chan exec.Elements, 20),
+		done:   make(chan struct{}),
+	}
+	if fromSource {
+		ec.want = 1 + int32(len(additionalTransforms))
+	}
+
+	// Just in case initial data for an instruction arrives *after* an instructon has ended.
+	// eg. it was blocked by another reader being slow, or the other instruction failed.
+	// So we provide a pre-completed reader, and do not cache it, as there's no further cleanup for it.
+	if _, ok := c.endedInstructions[id.instID]; ok {
+		// Since this is freshly created, we can set the close conditions immeadiately.
+		ec.got = 1
+		ec.want = ec.got
+		close(ec.ch)
+		return ec
+	}
+
+	c.channels[id.instID] = ec
+	return ec
+}
+
+// OpenTimerWrite returns io.WriteCloser for the given timerFamilyID, instruction and ptransform.
+func (c *DataChannel) OpenTimerWrite(ctx context.Context, ptransformID string, instID instructionID, family string) io.WriteCloser {
+	return c.makeTimerWriter(ctx, clientID{ptransformID: ptransformID, instID: instID}, family)
 }
 
 func (c *DataChannel) read(ctx context.Context) {
-	cache := make(map[clientID]*dataReader)
+	cache := make(map[instructionID]*elementsChan)
+	seenLast := make([]clientID, 0, 5)
 	for {
 		msg, err := c.client.Recv()
 		if err != nil {
@@ -240,19 +358,13 @@ func (c *DataChannel) read(ctx context.Context) {
 			c.mu.Lock()
 			c.readErr = err // prevent not yet opened readers from hanging.
 			// Readers must be closed from this goroutine, since we can't
-			// close the r.buf channels twice, or send on a closed channel.
-			// Any other approach is racy, and may cause one of the above
-			// panics.
-			for _, m := range c.readers {
-				for _, r := range m {
-					log.Errorf(ctx, "DataChannel.read %v reader %v closing due to error on channel", c.id, r.id)
-					if !r.completed {
-						r.completed = true
-						r.err = err
-						close(r.buf)
-					}
-					delete(cache, r.id)
+			// close the elementsChan channel twice, or send on those closed channels.
+			// Any other approach is racy, and may cause one of the above panics.
+			for instID, ec := range c.channels {
+				if !ec.Closed() {
+					close(ec.ch)
 				}
+				delete(cache, instID)
 			}
 			c.terminateStreamOnError(err)
 			c.mu.Unlock()
@@ -270,120 +382,92 @@ func (c *DataChannel) read(ctx context.Context) {
 		// Each message may contain segments for multiple streams, so we
 		// must treat each segment in isolation. We maintain a local cache
 		// to reduce lock contention.
-
-		for _, elm := range msg.GetData() {
-			id := clientID{ptransformID: elm.TransformId, instID: instructionID(elm.GetInstructionId())}
-
-			var r *dataReader
-			if local, ok := cache[id]; ok {
-				r = local
-			} else {
-				c.mu.Lock()
-				r = c.makeReader(ctx, id)
-				c.mu.Unlock()
-				cache[id] = r
-			}
-
-			if elm.GetIsLast() {
-				// If this reader hasn't closed yet, do so now.
-				if !r.completed {
-					// Use the last segment if any.
-					if len(elm.GetData()) != 0 {
-						// In case of local side closing, send with select.
-						select {
-						case r.buf <- elm.GetData():
-						case <-r.done:
-						}
-					}
-					// Close buffer to signal EOF.
-					r.completed = true
-					close(r.buf)
+		iterateElements(ctx, "timer", c, cache, &seenLast, msg.GetTimers(),
+			func(elm *fnpb.Elements_Timers) exec.Elements {
+				return exec.Elements{Timers: elm.GetTimers(), PtransformID: elm.GetTransformId(), TimerFamilyID: elm.GetTimerFamilyId()}
+			})
+
+		iterateElements(ctx, "data", c, cache, &seenLast, msg.GetData(),
+			func(elm *fnpb.Elements_Data) exec.Elements {
+				return exec.Elements{Data: elm.GetData(), PtransformID: elm.GetTransformId()}
+			})
+
+		// Mark all readers that we've seen the last of as done, after queuing their elements.
+		if len(seenLast) > 0 {
+			c.mu.Lock()
+			for _, id := range seenLast {
+				r, ok := cache[id.instID]
+				if !ok {
+					continue // we've already closed this cached reader, skip
+				}
+				r.PTransformDone()
+				if r.Closed() {
+					// Clean up local bookkeeping. We'll never see another message
+					// for it again. We have to be careful not to remove the real
+					// one, because readers may be initialized after we've seen
+					// the full stream.
+					delete(cache, id.instID)
 				}
-
-				// Clean up local bookkeeping. We'll never see another message
-				// for it again. We have to be careful not to remove the real
-				// one, because readers may be initialized after we've seen
-				// the full stream.
-				delete(cache, id)
-				continue
-			}
-
-			if r.completed {
-				// The local reader has closed but the remote is still sending data.
-				// Just ignore it. We keep the reader config in the cache so we don't
-				// treat it as a new reader. Eventually the stream will finish and go
-				// through normal teardown.
-				continue
-			}
-
-			// This send is deliberately blocking, if we exceed the buffering for
-			// a reader. We can't buffer the entire main input, if some user code
-			// is slow (or gets stuck). If the local side closes, the reader
-			// will be marked as completed and further remote data will be ignored.
-			select {
-			case r.buf <- elm.GetData():
-			case <-r.done:
-				r.completed = true
-				close(r.buf)
 			}
+			seenLast = seenLast[:0] // reset for re-use
+			c.mu.Unlock()
 		}
 	}
 }
 
-type errReader struct {
-	err error
-}
-
-func (r *errReader) Read(_ []byte) (int, error) {
-	return 0, r.err
-}
-
-func (r *errReader) Close() error {
-	return r.err
+// dataEle is a light interface against the proto Data and Timer Elements.
+type dataEle interface {
+	GetTransformId() string
+	GetInstructionId() string
+	GetIsLast() bool
 }
 
-// makeReader creates a dataReader. It expects to be called while c.mu is held.
-func (c *DataChannel) makeReader(ctx context.Context, id clientID) *dataReader {
-	var m map[string]*dataReader
-	var ok bool
-	if m, ok = c.readers[id.instID]; !ok {
-		m = make(map[string]*dataReader)
-		c.readers[id.instID] = m
-	}
-
-	if r, ok := m[id.ptransformID]; ok {
-		return r
-	}
-
-	r := &dataReader{id: id, buf: make(chan []byte, bufElements), done: make(chan bool, 1), channel: c}
+func iterateElements[E dataEle](ctx context.Context, kind string, c *DataChannel, cache map[instructionID]*elementsChan, seenLast *[]clientID, elms []E, wrap func(E) exec.Elements) {
+	for _, elm := range elms {
+		id := clientID{ptransformID: elm.GetTransformId(), instID: instructionID(elm.GetInstructionId())}
 
-	// Just in case initial data for an instruction arrives *after* an instructon has ended.
-	// eg. it was blocked by another reader being slow, or the other instruction failed.
-	// So we provide a pre-completed reader, and do not cache it, as there's no further cleanup for it.
-	if _, ok := c.endedInstructions[id.instID]; ok {
-		r.completed = true
-		close(r.buf)
-		r.err = io.EOF // In case of any actual data readers, so they terminate without error.
-		return r
-	}
+		var ec *elementsChan
+		if local, ok := cache[id.instID]; ok {
+			ec = local
+		} else {
+			c.mu.Lock()
+			ec = c.makeChannel(ctx, false, id)
+			c.mu.Unlock()
+			cache[id.instID] = ec
+		}
 
-	m[id.ptransformID] = r
-	return r
-}
+		if ec.Closed() {
+			continue
+		}
 
-func (c *DataChannel) removeReader(id clientID) {
-	c.mu.Lock()
-	if m, ok := c.readers[id.instID]; ok {
-		delete(m, id.ptransformID)
+		// This send deliberately blocks if we exceed the buffering for
+		// a reader. We can't buffer the entire main input, if some user code
+		// is slow (or gets stuck). If the local side closes, the reader
+		// will be marked as completed and further remote data will be ignored.
+		select {
+		case ec.ch <- wrap(elm):
+		case <-ec.done: // In case of out of band cancels.
+			ec.mu.Lock()
+			ec.got = ec.want
+			close(ec.ch)
+			ec.mu.Unlock()
+		case <-ctx.Done():
+			ec.mu.Lock()
+			ec.got = ec.want
+			close(ec.ch)
+			ec.mu.Unlock()
+		}
+		if elm.GetIsLast() {
+			*seenLast = append(*seenLast, id)
+		}
 	}
-	c.mu.Unlock()
 }
 
 const endedInstructionCap = 32
 
 // removeInstruction closes all readers and writers registered for the instruction
 // and deletes this instruction from the channel's reader and writer maps.
-func (c *DataChannel) removeInstruction(instID instructionID) {
+func (c *DataChannel) removeInstruction(instID instructionID) error {
 	c.mu.Lock()
 
 	// We don't want to leak memory, so cap the endedInstructions list.
@@ -395,21 +479,29 @@ func (c *DataChannel) removeInstruction(instID instructionID) {
 	c.endedInstructions[instID] = struct{}{}
 	c.rmQueue = append(c.rmQueue, instID)
 
-	rs := c.readers[instID]
 	ws := c.writers[instID]
+	tws := c.timerWriters[instID]
+	ec := c.channels[instID]
 
 	// Prevent other users while we iterate.
-	delete(c.readers, instID)
 	delete(c.writers, instID)
+	delete(c.timerWriters, instID)
+	delete(c.channels, instID)
+
+	// Return readErr to defend against data loss via short reads.
+	err := c.readErr
 	c.mu.Unlock()
 
-	// Close grabs the channel lock, so this must be outside the critical section.
-	for _, r := range rs {
-		r.Close()
-	}
 	for _, w := range ws {
 		w.Close()
 	}
+	for _, tw := range tws {
+		tw.Close()
+	}
+	if ec != nil {
+		ec.InstructionEnded()
+	}
+	return err
 }
 
 func (c *DataChannel) makeWriter(ctx context.Context, id clientID) *dataWriter {
@@ -436,46 +528,28 @@ func (c *DataChannel) makeWriter(ctx context.Context, id clientID) *dataWriter {
 	return w
 }
 
-type dataReader struct {
-	id        clientID
-	buf       chan []byte
-	done      chan bool
-	cur       []byte
-	channel   *DataChannel
-	completed bool
-	err       error
-}
-
-func (r *dataReader) Close() error {
-	r.done <- true
-	r.channel.removeReader(r.id)
-	return nil
-}
+func (c *DataChannel) makeTimerWriter(ctx context.Context, id clientID, family string) *timerWriter {
+	c.mu.Lock()
+	defer c.mu.Unlock()
 
-func (r *dataReader) Read(buf []byte) (int, error) {
-	if r.cur == nil {
-		b, ok := <-r.buf
-		if !ok {
-			if r.err == nil {
-				return 0, io.EOF
-			}
-			return 0, r.err
-		}
-		r.cur = b
+	var m map[timerKey]*timerWriter
+	var ok bool
+	if m, ok = c.timerWriters[id.instID]; !ok {
+		m = make(map[timerKey]*timerWriter)
+		c.timerWriters[id.instID] = m
 	}
-
-	// We don't need to check for a 0 length copy from r.cur here, since that's
-	// checked before buffers are handed to the r.buf channel.
-	n := copy(buf, r.cur)
-
-	switch {
-	case len(r.cur) == n:
-		r.cur = nil
-	default:
-		r.cur = r.cur[n:]
+	tk := timerKey{ptransformID: id.ptransformID, family: family}
+	if w, ok := m[tk]; ok {
+		return w
 	}
 
-	return n, nil
+	// We don't check for finished instructions for writers, as writers
+	// can only be created if an instruction is in scope, and aren't
+	// runner or user directed.
+
+	w := &timerWriter{ch: c, id: id, timerFamilyID: family}
+	m[tk] = w
+	return w
 }
 
 type dataWriter struct {
@@ -574,3 +648,75 @@ func (w *dataWriter) Write(p []byte) (n int, err error) {
 	w.buf = append(w.buf, p...)
 	return len(p), nil
 }
+
+type timerWriter struct {
+	id            clientID
+	timerFamilyID string
+	ch            *DataChannel
+}
+
+// send requires the ch.mu lock to be held.
+func (w *timerWriter) send(msg *fnpb.Elements) error {
+	recordStreamSend(msg)
+	if err := w.ch.client.Send(msg); err != nil {
+		if err == io.EOF {
+			log.Warnf(context.TODO(), "dataWriter[%v;%v] EOF on send; fetching real error", w.id, w.ch.id)
+			err = nil
+			for err == nil {
+				// Per GRPC stream documentation, if there's an EOF, we must call Recv
+				// until a non-nil error is returned, to ensure resources are cleaned up.
+				// https://pkg.go.dev/google.golang.org/grpc#ClientConn.NewStream
+				_, err = w.ch.client.Recv()
+			}
+		}
+		log.Warnf(context.TODO(), "dataWriter[%v;%v] error on send: %v", w.id, w.ch.id, err)
+		w.ch.terminateStreamOnError(err)
+		return err
+	}
+	return nil
+}
+
+func (w *timerWriter) Close() error {
+	w.ch.mu.Lock()
+	defer w.ch.mu.Unlock()
+	delete(w.ch.timerWriters[w.id.instID], timerKey{w.id.ptransformID, w.timerFamilyID})
+	var msg *fnpb.Elements
+	msg = &fnpb.Elements{
+		Timers: []*fnpb.Elements_Timers{
+			{
+				InstructionId: string(w.id.instID),
+				TransformId:   w.id.ptransformID,
+				TimerFamilyId: w.timerFamilyID,
+				IsLast:        true,
+			},
+		},
+	}
+	return w.send(msg)
+}
+
+func (w *timerWriter) writeTimers(p []byte) error {
+	w.ch.mu.Lock()
+	defer w.ch.mu.Unlock()
+
+	log.Infof(context.TODO(), "DEBUGLOG: timer write for %+v: %v", w.id, p)
+
+	msg := &fnpb.Elements{
+		Timers: []*fnpb.Elements_Timers{
+			{
+				InstructionId: string(w.id.instID),
+				TransformId:   w.id.ptransformID,
+				TimerFamilyId: w.timerFamilyID,
+				Timers:        p,
+			},
+		},
+	}
+	return w.send(msg)
+}
+
+func (w *timerWriter) Write(p []byte) (n int, err error) {
+	// write timers directly without buffering.
+	if err := w.writeTimers(p); err != nil {
+		return 0, err
+	}
+	return len(p), nil
+}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
index f69d9abde49..2bd31020c16 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
@@ -21,11 +21,13 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"runtime"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/exec"
 	fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1"
 )
 
@@ -101,102 +103,302 @@ func (f *fakeDataClient) Send(*fnpb.Elements) error {
 	return nil
 }
 
-func TestDataChannelTerminate_dataReader(t *testing.T) {
-	// The logging of channels closed is quite noisy for this test
-	log.SetOutput(io.Discard)
+type fakeChanClient struct {
+	ch  chan *fnpb.Elements
+	err error
+}
 
-	expectedError := fmt.Errorf("EXPECTED ERROR")
+func (f *fakeChanClient) Recv() (*fnpb.Elements, error) {
+	e, ok := <-f.ch
+	if !ok {
+		return nil, f.err
+	}
+	return e, nil
+}
+
+func (f *fakeChanClient) Send(e *fnpb.Elements) error {
+	f.ch <- e
+	return nil
+}
 
+func (f *fakeChanClient) Close() {
+	f.err = io.EOF
+	close(f.ch)
+}
+
+func (f *fakeChanClient) CloseWith(err error) {
+	f.err = err
+	close(f.ch)
+}
+
+func TestElementChan(t *testing.T) {
+	const instID = "inst_ref"
+	dataID := "dataTransform"
+	timerID := "timerTransform"
+	timerFamily := "timerFamily"
+	setupClient := func(t *testing.T) (context.Context, *fakeChanClient, *DataChannel) {
+		fmt.Println("TTTTT ", t.Name())
+		t.Helper()
+		client := &fakeChanClient{ch: make(chan *fnpb.Elements, bufElements)}
+		ctx, cancelFn := context.WithCancel(context.Background())
+		t.Cleanup(cancelFn)
+		t.Cleanup(func() { client.Close() })
+
+		c := makeDataChannel(ctx, "id", client, cancelFn)
+		return ctx, client, c
+	}
+	drainAndSum := func(t *testing.T, elms <-chan exec.Elements) (sum, count int) {
+		t.Helper()
+		for e := range elms { // only exits if data channel is closed.
+			if len(e.Data) != 0 {
+				sum += int(e.Data[0])
+				count++
+			}
+			if len(e.Timers) != 0 {
+				if e.TimerFamilyID != timerFamily {
+					t.Errorf("timer received without family set: %v, state= sum %v, count %v", e, sum, count)
+				}
+				sum += int(e.Timers[0])
+				count++
+			}
+		}
+		return sum, count
+	}
+
+	timerElm := func(val byte, isLast bool) *fnpb.Elements_Timers {
+		return &fnpb.Elements_Timers{InstructionId: instID, TransformId: timerID, Timers: []byte{val}, IsLast: isLast, TimerFamilyId: timerFamily}
+	}
+	dataElm := func(val byte, isLast bool) *fnpb.Elements_Data {
+		return &fnpb.Elements_Data{InstructionId: instID, TransformId: dataID, Data: []byte{val}, IsLast: isLast}
+	}
+	noTimerElm := func() *fnpb.Elements_Timers {
+		return &fnpb.Elements_Timers{InstructionId: instID, TransformId: timerID, Timers: []byte{}, IsLast: true}
+	}
+	noDataElm := func() *fnpb.Elements_Data {
+		return &fnpb.Elements_Data{InstructionId: instID, TransformId: dataID, Data: []byte{}, IsLast: true}
+	}
+	openChan := func(ctx context.Context, t *testing.T, c *DataChannel, timers ...string) <-chan exec.Elements {
+		t.Helper()
+		runtime.Gosched() // Encourage the "read" goroutine to schedule before this call, if necessary.
+		elms, err := c.OpenElementChan(ctx, dataID, instID, timers)
+		if err != nil {
+			t.Errorf("Unexpected error from OpenElementChan(%v, %v, nil): %v", dataID, instID, err)
+		}
+		return elms
+	}
+
+	// Most Cases
 	tests := []struct {
-		name          string
-		expectedError error
-		caseFn        func(t *testing.T, r io.ReadCloser, client *fakeDataClient, c *DataChannel)
+		name               string
+		sequenceFn         func(context.Context, *testing.T, *fakeChanClient, *DataChannel) <-chan exec.Elements
+		wantSum, wantCount int
 	}{
 		{
-			name:          "onClose",
-			expectedError: io.EOF,
-			caseFn: func(t *testing.T, r io.ReadCloser, client *fakeDataClient, c *DataChannel) {
-				// We don't read up all the buffered data, but immediately close the reader.
-				// Previously, since nothing was consuming the incoming gRPC data, the whole
-				// data channel would get stuck, and the client.Recv() call was eventually
-				// no longer called.
-				r.Close()
-
-				// If done is signaled, that means client.Recv() has been called to flush the
-				// channel, meaning consumer code isn't stuck.
-				<-client.done
+			name: "ReadThenData_singleRecv",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				elms := openChan(ctx, t, c)
+				client.Send(&fnpb.Elements{
+					Data: []*fnpb.Elements_Data{
+						dataElm(1, false),
+						dataElm(2, false),
+						dataElm(3, true),
+					},
+				})
+				return elms
 			},
+			wantSum: 6, wantCount: 3,
 		}, {
-			name:          "onSentinel",
-			expectedError: io.EOF,
-			caseFn: func(t *testing.T, r io.ReadCloser, client *fakeDataClient, c *DataChannel) {
-				// fakeDataClient eventually returns a sentinel element.
+			name: "ReadThenData_multipleRecv",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				elms := openChan(ctx, t, c)
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(2, false)}})
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
+				return elms
 			},
+			wantSum: 6, wantCount: 3,
 		}, {
-			name:          "onIsLast_withData",
-			expectedError: io.EOF,
-			caseFn: func(t *testing.T, r io.ReadCloser, client *fakeDataClient, c *DataChannel) {
-				// Set the last call with data to use is_last.
-				client.isLastCall = 2
+			name: "ReadThenNoData",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				elms := openChan(ctx, t, c)
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
+				return elms
 			},
+			wantSum: 0, wantCount: 0,
 		}, {
-			name:          "onIsLast_withoutData",
-			expectedError: io.EOF,
-			caseFn: func(t *testing.T, r io.ReadCloser, client *fakeDataClient, c *DataChannel) {
-				// Set the call without data to use is_last.
-				client.isLastCall = 3
+			name: "NoDataThenRead",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
+				elms := openChan(ctx, t, c)
+				return elms
 			},
+			wantSum: 0, wantCount: 0,
 		}, {
-			name:          "onRecvError",
-			expectedError: expectedError,
-			caseFn: func(t *testing.T, r io.ReadCloser, client *fakeDataClient, c *DataChannel) {
-				// The SDK starts reading in a goroutine immeadiately after open.
-				// Set the 2nd Recv call to have an error.
-				client.err = expectedError
+			name: "NoDataInstEndsThenRead",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
+				c.removeInstruction(instID)
+				elms := openChan(ctx, t, c)
+				return elms
 			},
+			wantSum: 0, wantCount: 0,
 		}, {
-			name:          "onInstructionEnd",
-			expectedError: io.EOF,
-			caseFn: func(t *testing.T, r io.ReadCloser, client *fakeDataClient, c *DataChannel) {
-				c.removeInstruction("inst_ref")
+			name: "ReadThenDataAndTimers",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				elms := openChan(ctx, t, c, timerID)
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+				client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}})
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
+				return elms
 			},
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			done := make(chan bool, 1)
-			client := &fakeDataClient{t: t, done: done}
-			ctx, cancelFn := context.WithCancel(context.Background())
-			c := makeDataChannel(ctx, "id", client, cancelFn)
+			wantSum: 6, wantCount: 3,
+		}, {
+			name: "AllDataAndTimersThenRead",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+				client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}})
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
+				elms := openChan(ctx, t, c, timerID)
+				return elms
+			},
+			wantSum: 6, wantCount: 3,
+		}, {
+			name: "FillBufferThenAbortThenRead",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				for i := 0; i < bufElements+2; i++ {
+					client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+				}
+				elms := openChan(ctx, t, c, timerID)
+				c.removeInstruction(instID)
 
-			r := c.OpenRead(ctx, "ptr", "inst_ref")
+				// These will be ignored
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(2, false)}})
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
+				return elms
+			},
+			wantSum: bufElements, wantCount: bufElements,
+		}, {
+			name: "DataThenReaderThenLast",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{
+					Data: []*fnpb.Elements_Data{
+						dataElm(1, false),
+						dataElm(2, false),
+					},
+				})
+				elms := openChan(ctx, t, c)
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
+				return elms
+			},
+			wantSum: 6, wantCount: 3,
+		}, {
+			name: "PartialTimersAllDataReadThenLastTimer",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{
+					Timers: []*fnpb.Elements_Timers{
+						timerElm(1, false),
+						timerElm(2, false),
+					},
+				})
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
+
+				elms := openChan(ctx, t, c, timerID)
+				client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(3, true)}})
+
+				return elms
+			},
+			wantSum: 6, wantCount: 3,
+		}, {
+			name: "AllTimerThenReaderThenDataClose",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{
+					Timers: []*fnpb.Elements_Timers{
+						timerElm(1, false),
+						timerElm(2, false),
+						timerElm(3, true),
+					},
+				})
+
+				elms := openChan(ctx, t, c, timerID)
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
+
+				return elms
+			},
+			wantSum: 6, wantCount: 3,
+		}, {
+			name: "NoTimersThenReaderThenNoData",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{noTimerElm()}})
+				elms := openChan(ctx, t, c, timerID)
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
+				return elms
+			},
+			wantSum: 0, wantCount: 0,
+		}, {
+			name: "SomeTimersThenReaderThenAData",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(1, false), timerElm(2, true)}})
+				elms := openChan(ctx, t, c, timerID)
+				client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
+				return elms
+			},
+			wantSum: 6, wantCount: 3,
+		}, {
+			name: "SomeTimersAndADataThenReader",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{
+					Timers: []*fnpb.Elements_Timers{timerElm(1, false), timerElm(2, true)},
+					Data:   []*fnpb.Elements_Data{dataElm(3, true)},
+				})
+				elms := openChan(ctx, t, c, timerID)
+				return elms
+			},
+			wantSum: 6, wantCount: 3,
+		}, {
+			name: "PartialReadThenEndInstruction",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{
+					Data: []*fnpb.Elements_Data{
+						dataElm(1, false),
+						dataElm(2, false),
+					},
+				})
+				elms := openChan(ctx, t, c)
+				var sum int
+				e := <-elms
+				sum += int(e.Data[0])
+				e = <-elms
+				sum += int(e.Data[0])
+
+				if got, want := sum, 3; got != want {
+					t.Errorf("got sum %v, want sum %v", got, want)
+				}
 
-			n, err := r.Read(make([]byte, 4))
-			if err != nil {
-				t.Errorf("Unexpected error from read: %v, read %d bytes.", err, n)
-			}
-			test.caseFn(t, r, client, c)
-			// Drain the reader.
-			i := 1 // For the earlier Read.
-			for err == nil {
-				read := make([]byte, 4)
-				_, err = r.Read(read)
-				i++
-			}
+				// Simulate a split, where the remaining buffer wouldn't be read further, and the instruction ends.
+				c.removeInstruction(instID)
 
-			if got, want := err, test.expectedError; got != want {
-				t.Errorf("Unexpected error from read %d: got %v, want %v", i, got, want)
-			}
-			// Verify that new readers return the same error on their reads after client.Recv is done.
-			if n, err := c.OpenRead(ctx, "ptr", "inst_ref").Read(make([]byte, 4)); err != test.expectedError {
-				t.Errorf("Unexpected error from read: got %v, want, %v read %d bytes.", err, test.expectedError, n)
-			}
+				// Instruction is ended, so further data for this instruction is ignored.
+				client.Send(&fnpb.Elements{
+					Data: []*fnpb.Elements_Data{
+						dataElm(3, false),
+						dataElm(4, true),
+					},
+				})
 
-			select {
-			case <-ctx.Done(): // Assert that the context must have been cancelled on read failures.
-				return
-			case <-time.After(time.Second * 5):
-				t.Fatal("context wasn't cancelled")
+				elms = openChan(ctx, t, c)
+				return elms
+			},
+			wantSum: 0, wantCount: 0,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, client, c := setupClient(t)
+			elms := test.sequenceFn(ctx, t, client, c)
+			sum, count := drainAndSum(t, elms)
+			if wantSum, wantCount := test.wantSum, test.wantCount; sum != wantSum || count != wantCount {
+				t.Errorf("got sum %v, count %v, want sum %v, count %v", sum, count, wantSum, wantCount)
 			}
 		})
 	}
@@ -213,16 +415,9 @@ func TestDataChannelRemoveInstruction_dataAfterClose(t *testing.T) {
 
 	client.blocked.Unlock()
 
-	r := c.OpenRead(ctx, "ptr", "inst_ref")
-
-	dr := r.(*dataReader)
-	if !dr.completed || dr.err != io.EOF {
-		t.Errorf("Expected a closed reader, but was still open: completed: %v, err: %v", dr.completed, dr.err)
-	}
-
-	n, err := r.Read(make([]byte, 4))
-	if err != io.EOF {
-		t.Errorf("Unexpected error from read: %v, read %d bytes.", err, n)
+	_, err := c.OpenElementChan(ctx, "ptr", "inst_ref", nil)
+	if err != nil {
+		t.Errorf("Unexpected error from read: %v,", err)
 	}
 }
 
@@ -234,7 +429,7 @@ func TestDataChannelRemoveInstruction_limitInstructionCap(t *testing.T) {
 
 	for i := 0; i < endedInstructionCap+10; i++ {
 		instID := instructionID(fmt.Sprintf("inst_ref%d", i))
-		c.OpenRead(ctx, "ptr", instID)
+		c.OpenElementChan(ctx, "ptr", instID, nil)
 		c.removeInstruction(instID)
 	}
 	if got, want := len(c.endedInstructions), endedInstructionCap; got != want {