You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2023/11/05 04:47:27 UTC

(beam) branch prismFusion created (now ee38ca1db27)

This is an automated email from the ASF dual-hosted git repository.

lostluck pushed a change to branch prismFusion
in repository https://gitbox.apache.org/repos/asf/beam.git


      at ee38ca1db27 [prism] Fusion

This branch includes the following new commits:

     new ee38ca1db27 [prism] Fusion

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(beam) 01/01: [prism] Fusion

Posted by lo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lostluck pushed a commit to branch prismFusion
in repository https://gitbox.apache.org/repos/asf/beam.git

commit ee38ca1db27eb96d225383148258f5a1fc544864
Author: Robert Burke <ro...@frantil.com>
AuthorDate: Sat Nov 4 17:35:02 2023 -0700

    [prism] Fusion
---
 sdks/go/pkg/beam/runners/direct/direct_test.go     |  98 +++++++
 sdks/go/pkg/beam/runners/prism/internal/execute.go |   6 -
 .../beam/runners/prism/internal/execute_test.go    | 105 ++++++-
 .../beam/runners/prism/internal/handlecombine.go   |  17 +-
 .../pkg/beam/runners/prism/internal/handlepardo.go |  37 ++-
 .../beam/runners/prism/internal/handlerunner.go    |  61 ++++-
 .../pkg/beam/runners/prism/internal/preprocess.go  | 302 +++++++++++++++++----
 .../beam/runners/prism/internal/preprocess_test.go |  78 +++---
 .../beam/runners/prism/internal/separate_test.go   |   3 +-
 sdks/go/pkg/beam/runners/prism/internal/stage.go   |  44 ++-
 .../runners/prism/internal/unimplemented_test.go   |   1 -
 11 files changed, 616 insertions(+), 136 deletions(-)

diff --git a/sdks/go/pkg/beam/runners/direct/direct_test.go b/sdks/go/pkg/beam/runners/direct/direct_test.go
index a8108580aa2..bcfce612da3 100644
--- a/sdks/go/pkg/beam/runners/direct/direct_test.go
+++ b/sdks/go/pkg/beam/runners/direct/direct_test.go
@@ -29,6 +29,7 @@ import (
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/metrics"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/register"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/filter"
 	"github.com/google/go-cmp/cmp"
 )
@@ -542,3 +543,100 @@ func TestMain(m *testing.M) {
 	beam.Init()
 	os.Exit(m.Run())
 }
+
+func init() {
+	// Basic Registration
+	// 	beam.RegisterFunction(identity)
+	// 	beam.RegisterType(reflect.TypeOf((*source)(nil)))
+	// 	beam.RegisterType(reflect.TypeOf((*discard)(nil)))
+
+	// Generic registration
+	register.Function2x0(identity)
+	register.DoFn2x0[[]byte, func(int)]((*source)(nil))
+	register.DoFn1x0[int]((*discard)(nil))
+	register.Emitter1[int]()
+}
+
+type source struct {
+	Count int
+}
+
+func (fn *source) ProcessElement(_ []byte, emit func(int)) {
+	for i := 0; i < fn.Count; i++ {
+		emit(i)
+	}
+}
+
+func identity(v int, emit func(int)) {
+	emit(v)
+}
+
+type discard struct {
+	processed int
+}
+
+func (fn *discard) ProcessElement(int) {
+	fn.processed++
+}
+
+// BenchmarkPipe checks basic throughput and exec overhead with everything registered.
+//
+// Just registered: ~700-900ns per call, 330B per DoFn, across 5 allocs per DoFn
+//
+// goos: linux
+// goarch: amd64
+// pkg: github.com/apache/beam/sdks/v2/go/pkg/beam/runners/direct
+// cpu: 12th Gen Intel(R) Core(TM) i7-1260P
+// BenchmarkPipe/dofns=0-16         	 1657698	       763.0 ns/op	  10.49 MB/s	       763.0 ns/elm	     320 B/op	       6 allocs/op
+// BenchmarkPipe/dofns=1-16         	  832784	      1294 ns/op	  12.37 MB/s	      1294 ns/elm	     656 B/op	      11 allocs/op
+// BenchmarkPipe/dofns=2-16         	  633345	      1798 ns/op	  13.35 MB/s	       899.0 ns/elm	     992 B/op	      16 allocs/op
+// BenchmarkPipe/dofns=3-16         	  471106	      2446 ns/op	  13.08 MB/s	       815.4 ns/elm	    1329 B/op	      21 allocs/op
+// BenchmarkPipe/dofns=5-16         	  340099	      3634 ns/op	  13.21 MB/s	       726.8 ns/elm	    2001 B/op	      31 allocs/op
+// BenchmarkPipe/dofns=10-16        	  183429	      6957 ns/op	  12.65 MB/s	       695.7 ns/elm	    3683 B/op	      56 allocs/op
+// BenchmarkPipe/dofns=100-16       	   17956	     65986 ns/op	  12.25 MB/s	       659.9 ns/elm	   33975 B/op	     506 allocs/op
+//
+// Optimized w/ Generic reg: ~200-300ns per call, 150B per DoFn, across 2 allocs per DoFn
+//
+// goos: linux
+// goarch: amd64
+// pkg: github.com/apache/beam/sdks/v2/go/pkg/beam/runners/direct
+// cpu: 12th Gen Intel(R) Core(TM) i7-1260P
+// BenchmarkPipe/dofns=0-16         	 9319206	       131.5 ns/op	  60.85 MB/s	       131.5 ns/elm	     152 B/op	       2 allocs/op
+// BenchmarkPipe/dofns=1-16         	 4465477	       268.3 ns/op	  59.63 MB/s	       268.3 ns/elm	     304 B/op	       3 allocs/op
+// BenchmarkPipe/dofns=2-16         	 2876710	       431.9 ns/op	  55.56 MB/s	       216.0 ns/elm	     456 B/op	       5 allocs/op
+// BenchmarkPipe/dofns=3-16         	 2096349	       562.1 ns/op	  56.93 MB/s	       187.4 ns/elm	     608 B/op	       7 allocs/op
+// BenchmarkPipe/dofns=5-16         	 1347927	       823.8 ns/op	  58.27 MB/s	       164.8 ns/elm	     912 B/op	      11 allocs/op
+// BenchmarkPipe/dofns=10-16        	  737594	      1590 ns/op	  55.36 MB/s	       159.0 ns/elm	    1672 B/op	      21 allocs/op
+// BenchmarkPipe/dofns=100-16       	   60728	     19696 ns/op	  41.02 MB/s	       197.0 ns/elm	   15357 B/op	     201 allocs/op
+func BenchmarkPipe(b *testing.B) {
+	makeBench := func(numDoFns int) func(b *testing.B) {
+		return func(b *testing.B) {
+			b.ReportAllocs()
+			b.SetBytes(8 * int64(numDoFns+1))
+
+			disc := &discard{}
+			p, s := beam.NewPipelineWithRoot()
+			imp := beam.Impulse(s)
+			src := beam.ParDo(s, &source{Count: b.N}, imp)
+			iden := src
+			for i := 0; i < numDoFns; i++ {
+				iden = beam.ParDo(s, identity, iden)
+			}
+			beam.ParDo0(s, disc, iden)
+			Execute(context.TODO(), p)
+			if disc.processed != b.N {
+				b.Fatalf("processed dodn't match bench number: got %v want %v", disc.processed, b.N)
+			}
+			d := b.Elapsed()
+			div := numDoFns
+			if div == 0 {
+				div = 1
+			}
+			div = div * b.N
+			b.ReportMetric(float64(d)/float64(div), "ns/elm")
+		}
+	}
+	for _, numDoFns := range []int{0, 1, 2, 3, 5, 10, 100} {
+		b.Run(fmt.Sprintf("dofns=%d", numDoFns), makeBench(numDoFns))
+	}
+}
diff --git a/sdks/go/pkg/beam/runners/prism/internal/execute.go b/sdks/go/pkg/beam/runners/prism/internal/execute.go
index c1ac6ea4488..5e07e161dd5 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/execute.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/execute.go
@@ -104,7 +104,6 @@ func makeWorker(env string, j *jobservices.Job) (*worker.W, error) {
 
 type transformExecuter interface {
 	ExecuteUrns() []string
-	ExecuteWith(t *pipepb.PTransform) string
 	ExecuteTransform(stageID, tid string, t *pipepb.PTransform, comps *pipepb.Components, watermark mtime.Time, data [][]byte) *worker.B
 }
 
@@ -166,11 +165,6 @@ func executePipeline(ctx context.Context, wks map[string]*worker.W, j *jobservic
 		urn := t.GetSpec().GetUrn()
 		stage.exe = proc.transformExecuters[urn]
 
-		// Stopgap until everythinng's moved to handlers.
-		stage.envID = t.GetEnvironmentId()
-		if stage.exe != nil {
-			stage.envID = stage.exe.ExecuteWith(t)
-		}
 		stage.ID = fmt.Sprintf("stage-%03d", i)
 		wk := wks[stage.envID]
 
diff --git a/sdks/go/pkg/beam/runners/prism/internal/execute_test.go b/sdks/go/pkg/beam/runners/prism/internal/execute_test.go
index 1a5ae7989a0..ce821bef898 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/execute_test.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/execute_test.go
@@ -37,7 +37,7 @@ import (
 	"github.com/apache/beam/sdks/v2/go/test/integration/primitives"
 )
 
-func initRunner(t *testing.T) {
+func initRunner(t testing.TB) {
 	t.Helper()
 	if *jobopts.Endpoint == "" {
 		s := jobservices.NewServer(0, internal.RunPipeline)
@@ -64,7 +64,7 @@ func execute(ctx context.Context, p *beam.Pipeline) (beam.PipelineResult, error)
 	return universal.Execute(ctx, p)
 }
 
-func executeWithT(ctx context.Context, t *testing.T, p *beam.Pipeline) (beam.PipelineResult, error) {
+func executeWithT(ctx context.Context, t testing.TB, p *beam.Pipeline) (beam.PipelineResult, error) {
 	t.Log("startingTest - ", t.Name())
 	s1 := rand.NewSource(time.Now().UnixNano())
 	r1 := rand.New(s1)
@@ -587,3 +587,104 @@ func init() {
 func TestMain(m *testing.M) {
 	ptest.MainWithDefault(m, "testlocal")
 }
+
+func init() {
+	// Basic Registration
+	// beam.RegisterFunction(identity)
+	// beam.RegisterType(reflect.TypeOf((*source)(nil)))
+	// beam.RegisterType(reflect.TypeOf((*discard)(nil)))
+
+	// Generic registration
+	register.Function2x0(identity)
+	register.DoFn2x0[[]byte, func(int)]((*source)(nil))
+	register.DoFn1x0[int]((*discard)(nil))
+	register.Emitter1[int]()
+}
+
+type source struct {
+	Count int
+}
+
+func (fn *source) ProcessElement(_ []byte, emit func(int)) {
+	for i := 0; i < fn.Count; i++ {
+		emit(i)
+	}
+}
+
+func identity(v int, emit func(int)) {
+	emit(v)
+}
+
+type discard struct {
+	processed int
+}
+
+func (fn *discard) ProcessElement(int) {
+	fn.processed++
+}
+
+// BenchmarkPipe checks basic throughput and exec overhead with everything registered.
+//
+// No fusion (all elements encoded) (generic registration):
+//
+//		~2000ns per call, 2000B per DoFn, across 22 allocs per DoFn
+//	 (using Basic regsitration adds 3 allocs per DoFn, a ~200 bytes, and ~200-400ns/elm)
+//
+// goos: linux
+// goarch: amd64
+// pkg: github.com/apache/beam/sdks/v2/go/pkg/beam/runners/direct
+// cpu: 12th Gen Intel(R) Core(TM) i7-1260P
+// BenchmarkPipe/dofns=0-16         	  885811	      1333 ns/op	      1333 ns/elm	    1993 B/op	      22 allocs/op
+// BenchmarkPipe/dofns=1-16         	  457683	      2636 ns/op	      2636 ns/elm	    3986 B/op	      44 allocs/op
+// BenchmarkPipe/dofns=2-16         	  283699	      3975 ns/op	      1988 ns/elm	    6138 B/op	      66 allocs/op
+// BenchmarkPipe/dofns=3-16         	  212767	      5689 ns/op	      1896 ns/elm	    8504 B/op	      88 allocs/op
+// BenchmarkPipe/dofns=5-16         	  121842	      8279 ns/op	      1656 ns/elm	   11994 B/op	     132 allocs/op
+// BenchmarkPipe/dofns=10-16        	   22059	     52877 ns/op	      5288 ns/elm	   30614 B/op	     443 allocs/op
+// BenchmarkPipe/dofns=100-16       	    6614	    166364 ns/op	      1664 ns/elm	  192961 B/op	    2261 allocs/op
+//
+// With fusion (generic registration):
+// ~200ns per call, 150B per DoFn, across 2 allocs per DoFn
+// AKA comparible to Direct Runner, as expected.
+//
+// goos: linux
+// goarch: amd64
+// pkg: github.com/apache/beam/sdks/v2/go/pkg/beam/runners/direct
+// cpu: 12th Gen Intel(R) Core(TM) i7-1260P
+// BenchmarkPipe/dofns=0-16         	   7660638	       145.8 ns/op	       145.8 ns/elm	     152 B/op	       2 allocs/op
+// BenchmarkPipe/dofns=1-16         	   3676358	       313.3 ns/op	       313.3 ns/elm	     304 B/op	       4 allocs/op
+// BenchmarkPipe/dofns=2-16         	   2242688	       507.4 ns/op	       253.7 ns/elm	     457 B/op	       6 allocs/op
+// BenchmarkPipe/dofns=3-16         	   1726969	       662.6 ns/op	       220.9 ns/elm	     610 B/op	       8 allocs/op
+// BenchmarkPipe/dofns=5-16         	   1198765	      1005 ns/op	       201.0 ns/elm	     915 B/op	      12 allocs/op
+// BenchmarkPipe/dofns=10-16        	    631459	      1874 ns/op	       187.4 ns/elm	    1679 B/op	      22 allocs/op
+// BenchmarkPipe/dofns=100-16       	     57926	     19890 ns/op	       198.9 ns/elm	   15660 B/op	     206 allocs/op
+func BenchmarkPipe(b *testing.B) {
+	initRunner(b)
+	makeBench := func(numDoFns int) func(b *testing.B) {
+		return func(b *testing.B) {
+			b.ReportAllocs()
+			disc := &discard{}
+			p, s := beam.NewPipelineWithRoot()
+			imp := beam.Impulse(s)
+			src := beam.ParDo(s, &source{Count: b.N}, imp)
+			iden := src
+			for i := 0; i < numDoFns; i++ {
+				iden = beam.ParDo(s, identity, iden)
+			}
+			beam.ParDo0(s, disc, iden)
+			_, err := execute(context.Background(), p)
+			if err != nil {
+				b.Fatal(err)
+			}
+			d := b.Elapsed()
+			div := numDoFns
+			if div == 0 {
+				div = 1
+			}
+			div = div * b.N
+			b.ReportMetric(float64(d)/float64(div), "ns/elm")
+		}
+	}
+	for _, numDoFns := range []int{0, 1, 2, 3, 5, 10, 100} {
+		b.Run(fmt.Sprintf("dofns=%d", numDoFns), makeBench(numDoFns))
+	}
+}
diff --git a/sdks/go/pkg/beam/runners/prism/internal/handlecombine.go b/sdks/go/pkg/beam/runners/prism/internal/handlecombine.go
index ff9bd1e1c88..3f31ad77fd5 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/handlecombine.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/handlecombine.go
@@ -58,10 +58,16 @@ func (*combine) PrepareUrns() []string {
 }
 
 // PrepareTransform returns lifted combines and removes the leaves if enabled. Otherwise returns nothing.
-func (h *combine) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) (*pipepb.Components, []string) {
+func (h *combine) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) prepareResult {
 	// If we aren't lifting, the "default impl" for combines should be sufficient.
 	if !h.config.EnableLifting {
-		return nil, nil
+		return prepareResult{
+			SubbedComps: &pipepb.Components{
+				Transforms: map[string]*pipepb.PTransform{
+					tid: t,
+				},
+			},
+		}
 	}
 
 	// To lift a combine, the spec should contain a CombinePayload.
@@ -197,7 +203,7 @@ func (h *combine) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipe
 			liftEID:    tform(liftEID, urns.TransformPreCombine, pcolInID, liftedNID, t.GetEnvironmentId()),
 			gbkEID:     tform(gbkEID, urns.TransformGBK, liftedNID, groupedNID, ""),
 			mergeEID:   tform(mergeEID, urns.TransformMerge, groupedNID, mergedNID, t.GetEnvironmentId()),
-			extractEID: tform(mergeEID, urns.TransformExtract, mergedNID, pcolOutID, t.GetEnvironmentId()),
+			extractEID: tform(extractEID, urns.TransformExtract, mergedNID, pcolOutID, t.GetEnvironmentId()),
 		},
 	}
 
@@ -205,5 +211,8 @@ func (h *combine) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipe
 	// TODO recurse through sub transforms to remove?
 	// We don't need to remove the composite, since we don't add it in
 	// when we return the new transforms, so it's not in the topology.
-	return newComps, t.GetSubtransforms()
+	return prepareResult{
+		SubbedComps:   newComps,
+		RemovedLeaves: t.GetSubtransforms(),
+	}
 }
diff --git a/sdks/go/pkg/beam/runners/prism/internal/handlepardo.go b/sdks/go/pkg/beam/runners/prism/internal/handlepardo.go
index 2ac5ca5bbf5..e9d422ca107 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/handlepardo.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/handlepardo.go
@@ -58,7 +58,7 @@ func (*pardo) PrepareUrns() []string {
 
 // PrepareTransform handles special processing with respect to ParDos, since their handling is dependant on supported features
 // and requirements.
-func (h *pardo) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) (*pipepb.Components, []string) {
+func (h *pardo) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) prepareResult {
 
 	// ParDos are a pain in the butt.
 	// Combines, by comparison, are dramatically simpler.
@@ -89,21 +89,22 @@ func (h *pardo) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb
 		// so they're not included here. Any nearly any ParDo can have them.
 
 		// At their simplest, we don't need to do anything special at pre-processing time, and simply pass through as normal.
-		return &pipepb.Components{
-			Transforms: map[string]*pipepb.PTransform{
-				tid: t,
+		return prepareResult{
+			SubbedComps: &pipepb.Components{
+				Transforms: map[string]*pipepb.PTransform{
+					tid: t,
+				},
 			},
-		}, nil
+		}
 	}
 
 	// Side inputs add to topology and make fusion harder to deal with
 	// (side input producers can't be in the same stage as their consumers)
-	// But we don't have fusion yet, so no worries.
 
 	// State, Timers, Stable Input, Time Sorted Input, and some parts of SDF
-	// Are easier to deal including a fusion break. But We can do that with a
-	// runner specific transform for stable input, and another for timesorted
-	// input.
+	// Are easier to deal with by including a fusion break. But we can do that with a
+	// runner specific transform for stable input, and another for time sorted input.
+	// TODO add
 
 	// SplittableDoFns have 3 required phases and a 4th optional phase.
 	//
@@ -235,10 +236,16 @@ func (h *pardo) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb
 			EnvironmentId: t.GetEnvironmentId(),
 		},
 	}
-
-	return &pipepb.Components{
-		Coders:       coders,
-		Pcollections: pcols,
-		Transforms:   tforms,
-	}, t.GetSubtransforms()
+	return prepareResult{
+		SubbedComps: &pipepb.Components{
+			Coders:       coders,
+			Pcollections: pcols,
+			Transforms:   tforms,
+		},
+		RemovedLeaves: t.GetSubtransforms(),
+		// Force ProcessSized to be a root to ensure SDFs are able to split
+		// between elements or within elements.
+		// Also this is where a transform would be stateful anyway.
+		ForcedRoots: []string{eProcessID},
+	}
 }
diff --git a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
index 3f699e47e67..59e92675482 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
@@ -67,13 +67,55 @@ func (*runner) ConfigCharacteristic() reflect.Type {
 var _ transformPreparer = (*runner)(nil)
 
 func (*runner) PrepareUrns() []string {
-	return []string{urns.TransformReshuffle}
+	return []string{urns.TransformReshuffle, urns.TransformFlatten}
 }
 
 // PrepareTransform handles special processing with respect runner transforms, like reshuffle.
-func (h *runner) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) (*pipepb.Components, []string) {
+func (h *runner) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) prepareResult {
+	switch t.GetSpec().GetUrn() {
+	case urns.TransformFlatten:
+		return h.handleFlatten(tid, t, comps)
+	case urns.TransformReshuffle:
+		return h.handleReshuffle(tid, t, comps)
+	default:
+		panic("unknown urn to Prepare: " + t.GetSpec().GetUrn())
+	}
+}
+
+func (h *runner) handleFlatten(tid string, t *pipepb.PTransform, comps *pipepb.Components) prepareResult {
+	if !h.config.SDKFlatten {
+		t.EnvironmentId = ""         // force the flatten to be a runner transform due to configuration.
+		forcedRoots := []string{tid} // Have runner side transforms be roots.
+
+		// Force runner flatten consumers to be roots.
+		// This resolves merges between two runner transforms trying
+		// to execute together.
+		outColID := getOnlyValue(t.GetOutputs())
+		for ctid, t := range comps.GetTransforms() {
+			for _, gi := range t.GetInputs() {
+				if gi == outColID {
+					forcedRoots = append(forcedRoots, ctid)
+				}
+			}
+		}
+
+		// Return the new components which is the transforms consumer
+		return prepareResult{
+			// We sub this flatten with itself, to not drop it.
+			SubbedComps: &pipepb.Components{
+				Transforms: map[string]*pipepb.PTransform{
+					tid: t,
+				},
+			},
+			RemovedLeaves: nil,
+			ForcedRoots:   forcedRoots,
+		}
+	}
+	return prepareResult{}
+}
+
+func (h *runner) handleReshuffle(tid string, t *pipepb.PTransform, comps *pipepb.Components) prepareResult {
 	// TODO: Implement the windowing strategy the "backup" transforms used for Reshuffle.
-	// TODO: Implement a fusion break for reshuffles.
 
 	if h.config.SDKReshuffle {
 		panic("SDK side reshuffle not yet supported")
@@ -106,12 +148,15 @@ func (h *runner) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipep
 
 	// We need to remove the consumers of the output PCollection.
 	toRemove := []string{}
+	// We need to force the consumers to be stage root,
+	// because reshuffle should be a fusion break.
+	forcedRoots := []string{}
 
-	for _, t := range comps.GetTransforms() {
+	for tid, t := range comps.GetTransforms() {
 		for li, gi := range t.GetInputs() {
 			if gi == outColID {
-				// The whole s
 				t.GetInputs()[li] = inColID
+				forcedRoots = append(forcedRoots, tid)
 			}
 		}
 	}
@@ -120,7 +165,11 @@ func (h *runner) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipep
 	toRemove = append(toRemove, t.GetSubtransforms()...)
 
 	// Return the new components which is the transforms consumer
-	return nil, toRemove
+	return prepareResult{
+		SubbedComps:   nil, // Replace the reshuffle with nothing.
+		RemovedLeaves: toRemove,
+		ForcedRoots:   forcedRoots,
+	}
 }
 
 var _ transformExecuter = (*runner)(nil)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/preprocess.go b/sdks/go/pkg/beam/runners/prism/internal/preprocess.go
index bca40709626..55065929406 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/preprocess.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/preprocess.go
@@ -23,6 +23,7 @@ import (
 	pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns"
 	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 	"golang.org/x/exp/slog"
 	"google.golang.org/protobuf/encoding/prototext"
 )
@@ -35,7 +36,13 @@ type transformPreparer interface {
 	PrepareUrns() []string
 	// PrepareTransform takes a PTransform proto and returns a set of new Components, and a list of
 	// transformIDs leaves to remove and ignore from graph processing.
-	PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) (*pipepb.Components, []string)
+	PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) prepareResult
+}
+
+type prepareResult struct {
+	SubbedComps   *pipepb.Components
+	RemovedLeaves []string
+	ForcedRoots   []string
 }
 
 // preprocessor retains configuration for preprocessing the
@@ -73,6 +80,7 @@ func (p *preprocessor) preProcessGraph(comps *pipepb.Components) []*stage {
 	// TODO move this out of this part of the pre-processor?
 	leaves := map[string]struct{}{}
 	ignore := map[string]struct{}{}
+	forcedRoots := map[string]bool{}
 	for tid, t := range ts {
 		if _, ok := ignore[tid]; ok {
 			continue
@@ -106,30 +114,33 @@ func (p *preprocessor) preProcessGraph(comps *pipepb.Components) []*stage {
 			continue
 		}
 
-		subs, toRemove := h.PrepareTransform(tid, t, comps)
+		prepResult := h.PrepareTransform(tid, t, comps)
 
 		// Clear out unnecessary leaves from this composite for topological sort handling.
-		for _, key := range toRemove {
+		for _, key := range prepResult.RemovedLeaves {
 			ignore[key] = struct{}{}
 			delete(leaves, key)
 		}
+		for _, key := range prepResult.ForcedRoots {
+			forcedRoots[key] = true
+		}
 
 		// ts should be a clone, so we should be able to add new transforms into the map.
-		for tid, t := range subs.GetTransforms() {
+		for tid, t := range prepResult.SubbedComps.GetTransforms() {
 			leaves[tid] = struct{}{}
 			ts[tid] = t
 		}
-		for cid, c := range subs.GetCoders() {
+		for cid, c := range prepResult.SubbedComps.GetCoders() {
 			comps.GetCoders()[cid] = c
 		}
-		for nid, n := range subs.GetPcollections() {
+		for nid, n := range prepResult.SubbedComps.GetPcollections() {
 			comps.GetPcollections()[nid] = n
 		}
 		// It's unlikely for these to change, but better to handle them now, to save a headache later.
-		for wid, w := range subs.GetWindowingStrategies() {
+		for wid, w := range prepResult.SubbedComps.GetWindowingStrategies() {
 			comps.GetWindowingStrategies()[wid] = w
 		}
-		for envid, env := range subs.GetEnvironments() {
+		for envid, env := range prepResult.SubbedComps.GetEnvironments() {
 			comps.GetEnvironments()[envid] = env
 		}
 	}
@@ -141,6 +152,28 @@ func (p *preprocessor) preProcessGraph(comps *pipepb.Components) []*stage {
 	topological := pipelinex.TopologicalSort(ts, keptLeaves)
 	slog.Debug("topological transform ordering", slog.Any("topological", topological))
 
+	facts := computePColFacts(topological, comps)
+	facts.forcedRoots = forcedRoots
+
+	return greedyFusion(topological, comps, facts)
+}
+
+// TODO(lostluck): Be able to toggle this in variants.
+// Most likely, re-implement in terms of simply marking all transforms as forced roots.
+
+// defaultFusion is the base strategy for prism, that doesn't seek to optimize execution
+// with fused stages. Input is the set of leaf nodes we're going to execute, topologically
+// sorted, and the pipeline components.
+//
+// Default fusion behavior: Don't. Prism is intended to test all of Beam, which often
+// means for testing purposes, to execute pipelines without optimization.
+//
+// Special Exception to unfused Go SDK pipelines.
+//
+// If a transform, after a GBK step, has a single input with a KV<K, Iter<X>> coder
+// and a single output O with a KV<K, Iter<Y>> coder, and if then it must be fused with
+// the consumers of O.
+func defaultFusion(topological []string, comps *pipepb.Components, facts fusionFacts) []*stage {
 	// Basic Fusion Behavior
 	//
 	// Fusion is the practice of executing associated DoFns in the same stage.
@@ -172,29 +205,6 @@ func (p *preprocessor) preProcessGraph(comps *pipepb.Components) []*stage {
 	// stages. Beam supports this naturally with the separation of work into independant
 	// bundles for execution.
 
-	return defaultFusion(topological, comps)
-}
-
-// defaultFusion is the base strategy for prism, that doesn't seek to optimize execution
-// with fused stages. Input is the set of leaf nodes we're going to execute, topologically
-// sorted, and the pipeline components.
-//
-// Default fusion behavior: Don't. Prism is intended to test all of Beam, which often
-// means for testing purposes, to execute pipelines without optimization.
-//
-// Special Exception to unfused Go SDK pipelines.
-//
-// If a transform, after a GBK step, has a single input with a KV<K, Iter<X>> coder
-// and a single output O with a KV<K, Iter<Y>> coder, and if then it must be fused with
-// the consumers of O.
-func defaultFusion(topological []string, comps *pipepb.Components) []*stage {
-	var stages []*stage
-
-	// TODO figure out a better place to source the PCol Parents/Consumers analysis
-	// so we don't keep repeating it.
-
-	pcolParents, pcolConsumers := computPColFacts(topological, comps)
-
 	// Explicitly list the pcollectionID we want to fuse along.
 	fuseWithConsumers := map[string]string{}
 	for _, tid := range topological {
@@ -207,7 +217,7 @@ func defaultFusion(topological []string, comps *pipepb.Components) []*stage {
 		inputID := getOnlyValue(t.GetInputs())
 		outputID := getOnlyValue(t.GetOutputs())
 
-		parentLink := pcolParents[inputID]
+		parentLink := facts.pcolParents[inputID]
 
 		parent := comps.GetTransforms()[parentLink.transform]
 
@@ -225,6 +235,7 @@ func defaultFusion(topological []string, comps *pipepb.Components) []*stage {
 		}
 	}
 
+	var stages []*stage
 	// Since we iterate in topological order, we're guaranteed to process producers before consumers.
 	consumed := map[string]bool{} // Checks if we've already handled a transform already due to fusion.
 	for _, tid := range topological {
@@ -243,7 +254,7 @@ func defaultFusion(topological []string, comps *pipepb.Components) []*stage {
 		if !ok {
 			continue
 		}
-		cs := pcolConsumers[pcolID]
+		cs := facts.pcolConsumers[pcolID]
 
 		for _, c := range cs {
 			stg.transforms = append(stg.transforms, c.transform)
@@ -252,30 +263,76 @@ func defaultFusion(topological []string, comps *pipepb.Components) []*stage {
 	}
 
 	for _, stg := range stages {
-		prepareStage(stg, comps, pcolConsumers)
+		prepareStage(stg, comps, facts)
 	}
 	return stages
 }
 
-// computPColFacts computes a map of PCollectionIDs to their parent transforms, and a map of
+type fusionFacts struct {
+	pcolParents          map[string]link
+	pcolConsumers        map[string][]link
+	usedAsSideInput      map[string]bool
+	directSideInputs     map[string]map[string]bool
+	downstreamSideInputs map[string]map[string]bool
+	forcedRoots          map[string]bool
+}
+
+// computePColFacts computes a map of PCollectionIDs to their parent transforms, and a map of
 // PCollectionIDs to their consuming transforms.
-func computPColFacts(topological []string, comps *pipepb.Components) (map[string]link, map[string][]link) {
-	pcolParents := map[string]link{}
-	pcolConsumers := map[string][]link{}
+func computePColFacts(topological []string, comps *pipepb.Components) fusionFacts {
+	ret := fusionFacts{
+		pcolParents:          map[string]link{},
+		pcolConsumers:        map[string][]link{},
+		usedAsSideInput:      map[string]bool{},
+		directSideInputs:     map[string]map[string]bool{}, // direct set
+		downstreamSideInputs: map[string]map[string]bool{}, // transitive set
+	}
 
 	// Use the topological ids so each PCollection only has a single
 	// parent. We've already pruned out composites at this stage.
 	for _, tID := range topological {
 		t := comps.GetTransforms()[tID]
 		for local, global := range t.GetOutputs() {
-			pcolParents[global] = link{transform: tID, local: local, global: global}
+			ret.pcolParents[global] = link{transform: tID, local: local, global: global}
 		}
+		sis, err := getSideInputs(t)
+		if err != nil {
+			panic(err)
+		}
+		directSIs := map[string]bool{}
+		ret.directSideInputs[tID] = directSIs
 		for local, global := range t.GetInputs() {
-			pcolConsumers[global] = append(pcolConsumers[global], link{transform: tID, local: local, global: global})
+			ret.pcolConsumers[global] = append(ret.pcolConsumers[global], link{transform: tID, local: local, global: global})
+			if _, ok := sis[local]; ok {
+				ret.usedAsSideInput[global] = true
+				directSIs[global] = true
+			}
 		}
 	}
 
-	return pcolParents, pcolConsumers
+	for _, tID := range topological {
+		computeDownstreamSideInputs(tID, comps, ret)
+	}
+
+	return ret
+}
+
+func computeDownstreamSideInputs(tID string, comps *pipepb.Components, facts fusionFacts) map[string]bool {
+	if dssi, ok := facts.downstreamSideInputs[tID]; ok {
+		return dssi
+	}
+	dssi := map[string]bool{}
+	for _, o := range comps.GetTransforms()[tID].GetOutputs() {
+		if facts.usedAsSideInput[o] {
+			dssi[o] = true
+		}
+		for _, consumer := range facts.pcolConsumers[o] {
+			cdssi := computeDownstreamSideInputs(consumer.global, comps, facts)
+			maps.Copy(dssi, cdssi)
+		}
+	}
+	facts.downstreamSideInputs[tID] = dssi
+	return dssi
 }
 
 // We need to see that both coders have this pattern: KV<K, Iter<?>>
@@ -309,7 +366,7 @@ func checkForExpandCoderPattern(in, out string, comps *pipepb.Components) bool {
 // 1. Determining the single parallel input (may be 0 for impulse stages).
 // 2. Determining all outputs to the stages.
 // 3. Determining all side inputs.
-// 4  validating that no side input is fed by an internal PCollection.
+// 4  Validating that no side input is fed by an internal PCollection.
 // 4. Check that all transforms are in the same environment or are environment agnostic. (TODO for xlang)
 // 5. Validate that only the primary input consuming transform are stateful. (Might be able to relax this)
 //
@@ -320,9 +377,9 @@ func checkForExpandCoderPattern(in, out string, comps *pipepb.Components) bool {
 // Finally, it takes this information and caches it in the stage for simpler descriptor construction downstream.
 //
 // Note, this is very similar to the work done WRT composites in pipelinex.Normalize.
-func prepareStage(stg *stage, comps *pipepb.Components, pipelineConsumers map[string][]link) {
+func prepareStage(stg *stage, comps *pipepb.Components, pipelineFacts fusionFacts) {
 	// Collect all PCollections involved in this stage.
-	pcolParents, pcolConsumers := computPColFacts(stg.transforms, comps)
+	stageFacts := computePColFacts(stg.transforms, comps)
 
 	transformSet := map[string]bool{}
 	for _, tid := range stg.transforms {
@@ -333,9 +390,9 @@ func prepareStage(stg *stage, comps *pipepb.Components, pipelineConsumers map[st
 	mainInputs := map[string]string{}
 	var sideInputs []link
 	inputs := map[string]bool{}
-	for pid, plinks := range pcolConsumers {
+	for pid, plinks := range stageFacts.pcolConsumers {
 		// Check if this PCollection is generated in this bundle.
-		if _, ok := pcolParents[pid]; ok {
+		if _, ok := stageFacts.pcolParents[pid]; ok {
 			// It is, so we will ignore for now.
 			continue
 		}
@@ -354,10 +411,10 @@ func prepareStage(stg *stage, comps *pipepb.Components, pipelineConsumers map[st
 	outputs := map[string]link{}
 	var internal []string
 	// Look at all PCollections produced in this stage.
-	for pid, link := range pcolParents {
+	for pid, link := range stageFacts.pcolParents {
 		// Look at all consumers of this PCollection in the pipeline
 		isInternal := true
-		for _, l := range pipelineConsumers[pid] {
+		for _, l := range pipelineFacts.pcolConsumers[pid] {
 			// If the consuming transform isn't in the stage, it's an output.
 			if !transformSet[l.transform] {
 				isInternal = false
@@ -384,10 +441,151 @@ func prepareStage(stg *stage, comps *pipepb.Components, pipelineConsumers map[st
 	if l := len(mainInputs); l == 1 {
 		stg.primaryInput = getOnlyValue(mainInputs)
 	} else if l > 1 {
-		// Quick check that this is a lone flatten node, which is handled runner side anyway
-		// and only sent SDK side as part of a fused stage.
-		if !(len(stg.transforms) == 1 && comps.GetTransforms()[stg.transforms[0]].GetSpec().GetUrn() == urns.TransformFlatten) {
-			panic("expected flatten node, but wasn't")
+		// Quick check that this is lead by a flatten node, and that it's handled runner side.
+		t := comps.GetTransforms()[stg.transforms[0]]
+		if !(t.GetSpec().GetUrn() == urns.TransformFlatten && t.GetEnvironmentId() == "") {
+			panic("expected runner flatten node, but wasn't")
 		}
 	}
 }
+
+// greedyFusion produces a pipeline as tightly fused as possible.
+//
+// Fusion is a critical optimization for performance of pipeline execution.
+// Thus it's important for SDKs to be capable of executing transforms in a fused state.
+//
+// However, not all transforms can be fused into the same stage together.
+// Further, some transforms must be at the root of a stage.
+//
+// # Fusion Restrictions
+//
+// Environments: Transforms that aren't in the same environment can't b
+// fused together *unless* their environments can also be fused together.
+// Eg. Resource hints can often be ignored for local runners.
+//
+// Side Inputs: A transform S consuming a PCollection as a side input can't
+// be fused with the  transform P that produces that PCollection. Further,
+// no transform S+ descended from S, can be fused with transform P.
+//
+// Splittable DoFns: An expanded Splittable DoFn transform's Process Sized
+// Elements and Restrictions component must be the root of a stage.
+//
+// State and Timers: Stateful Transforms (transforms using State and Timers)
+// must be the root of transforms, since they are required to be keyed.
+// A sequence of Key Preserving stateful transforms could be fused.
+//
+// TODO: Sink/Unzip Flattens so they vanish from the graph.
+//
+// This approach is largely cribed from the Python approach at
+// fn_api_runner/translations.py. That implementation is very set oriented &
+// eagerly adds data source/sink transforms, while prism does so later in
+// stage construction.
+func greedyFusion(topological []string, comps *pipepb.Components, facts fusionFacts) []*stage {
+	fused := map[int]int{}
+	stageAssignments := map[string]int{}
+
+	stageEnvs := map[int]string{}
+	forcedRoots := map[int]bool{}
+	directSIs := map[int]map[string]bool{}
+	downstreamSIs := map[int]map[string]bool{}
+	pcolParents := map[string]int{}
+
+	var index int
+	replacements := func(tID string) int {
+		sID, ok := stageAssignments[tID]
+		if !ok { // No stage exists yet.
+			sID = index
+			index++
+
+			t := comps.GetTransforms()[tID]
+			stageAssignments[tID] = sID
+			stageEnvs[sID] = t.GetEnvironmentId()
+			forcedRoots[sID] = facts.forcedRoots[tID]
+			directSIs[sID] = maps.Clone(facts.directSideInputs[tID])
+			downstreamSIs[sID] = maps.Clone(facts.downstreamSideInputs[tID])
+		}
+
+		var oldIDs []int
+		rep, ok := fused[sID]
+		for ok {
+			oldIDs = append(oldIDs, sID)
+			sID = rep
+			rep, ok = fused[sID]
+		}
+		// Update the assignment & fusions for path shortening.
+		stageAssignments[tID] = sID
+		for _, old := range oldIDs {
+			fused[old] = sID
+		}
+		return sID
+	}
+
+	overlap := func(downstream, consumer map[string]bool) bool {
+		for si := range consumer {
+			if downstream[si] {
+				return true
+			}
+		}
+		return false
+	}
+
+	// To start, every transform is in it's own stage.
+	// So we map a transformID to a stageID.
+	// We go through each PCollection, (facts.PcolParents) and
+	// try to fuse the producer to each consumer of that PCollection.
+	//
+	// If we can fuse, the consumer takes on the producer's stageID,
+	// and the assignments are updated.
+
+	// Use the topological sort instead?
+
+	keys := maps.Keys(facts.pcolParents)
+	slices.Sort(keys)
+	for _, pcol := range keys {
+		producer := facts.pcolParents[pcol]
+		for _, consumer := range facts.pcolConsumers[pcol] {
+			pID := replacements(producer.transform) // Get current stage for producer
+			pcolParents[pcol] = pID
+			cID := replacements(consumer.transform) // Get current stage for consumer
+
+			// See if there's anything preventing fusion:
+			if pID == cID {
+				continue // Already fused together.
+			}
+			if stageEnvs[pID] != stageEnvs[cID] {
+				continue // Not the same environment.
+			}
+			if forcedRoots[cID] {
+				continue // Forced root.
+			}
+			if overlap(downstreamSIs[pID], directSIs[cID]) {
+				continue // Side input conflict
+			}
+
+			// In principle, we can fuse!
+			fused[cID] = pID // Set the consumer to be in the producer's stage.
+			// Copy the consumer's direct and downstream side input sets into the producer.
+			maps.Copy(directSIs[pID], directSIs[cID])
+			maps.Copy(downstreamSIs[pID], downstreamSIs[cID])
+		}
+	}
+
+	var stages []*stage
+	fusedToStages := map[int]*stage{}
+	for _, tID := range topological {
+		sID := replacements(tID)
+		s := fusedToStages[sID]
+		if s == nil {
+			s = &stage{
+				envID: stageEnvs[sID],
+			}
+			fusedToStages[sID] = s
+			stages = append(stages, s)
+		}
+		s.transforms = append(s.transforms, tID)
+	}
+	for _, stg := range stages {
+		prepareStage(stg, comps, facts)
+	}
+	return stages
+}
diff --git a/sdks/go/pkg/beam/runners/prism/internal/preprocess_test.go b/sdks/go/pkg/beam/runners/prism/internal/preprocess_test.go
index ba39d024e71..8d0d6accdf6 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/preprocess_test.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/preprocess_test.go
@@ -31,6 +31,7 @@ func Test_preprocessor_preProcessGraph(t *testing.T) {
 
 		wantComponents *pipepb.Components
 		wantStages     []*stage
+		forcedRoots    []string
 	}{
 		{
 			name: "noPreparer",
@@ -57,7 +58,8 @@ func Test_preprocessor_preProcessGraph(t *testing.T) {
 				},
 			},
 		}, {
-			name: "preparer",
+			name:        "preparer",
+			forcedRoots: []string{"e1_early", "e1_late"},
 			input: &pipepb.Components{
 				Transforms: map[string]*pipepb.PTransform{
 					"e1": {
@@ -125,7 +127,9 @@ func Test_preprocessor_preProcessGraph(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			pre := newPreprocessor([]transformPreparer{&testPreparer{}})
+			pre := newPreprocessor([]transformPreparer{&testPreparer{
+				ForcedRoots: test.forcedRoots,
+			}})
 
 			gotStages := pre.preProcessGraph(test.input)
 			if diff := cmp.Diff(test.wantStages, gotStages, cmp.AllowUnexported(stage{}, link{}), cmpopts.EquateEmpty()); diff != "" {
@@ -139,47 +143,53 @@ func Test_preprocessor_preProcessGraph(t *testing.T) {
 	}
 }
 
-type testPreparer struct{}
+type testPreparer struct {
+	ForcedRoots []string
+}
 
 func (p *testPreparer) PrepareUrns() []string {
 	return []string{"test_urn"}
 }
 
-func (p *testPreparer) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) (*pipepb.Components, []string) {
-	return &pipepb.Components{
-		Transforms: map[string]*pipepb.PTransform{
-			"e1_early": {
-				UniqueName: "e1_early",
-				Spec: &pipepb.FunctionSpec{
-					Urn: "defaultUrn",
+func (p *testPreparer) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) prepareResult {
+	return prepareResult{
+		ForcedRoots: p.ForcedRoots,
+		SubbedComps: &pipepb.Components{
+			Transforms: map[string]*pipepb.PTransform{
+				"e1_early": {
+					UniqueName: "e1_early",
+					Spec: &pipepb.FunctionSpec{
+						Urn: "defaultUrn",
+					},
+					Outputs:       map[string]string{"i0": "pcol1"},
+					EnvironmentId: "env1",
+				},
+				"e1_late": {
+					UniqueName: "e1_late",
+					Spec: &pipepb.FunctionSpec{
+						Urn: "defaultUrn",
+					},
+					Inputs:        map[string]string{"i0": "pcol1"},
+					EnvironmentId: "env1",
 				},
-				Outputs:       map[string]string{"i0": "pcol1"},
-				EnvironmentId: "env1",
 			},
-			"e1_late": {
-				UniqueName: "e1_late",
-				Spec: &pipepb.FunctionSpec{
-					Urn: "defaultUrn",
+			Pcollections: map[string]*pipepb.PCollection{
+				"pcol1": {
+					UniqueName:          "pcol1",
+					CoderId:             "coder1",
+					WindowingStrategyId: "ws1",
 				},
-				Inputs:        map[string]string{"i0": "pcol1"},
-				EnvironmentId: "env1",
 			},
-		},
-		Pcollections: map[string]*pipepb.PCollection{
-			"pcol1": {
-				UniqueName:          "pcol1",
-				CoderId:             "coder1",
-				WindowingStrategyId: "ws1",
+			Coders: map[string]*pipepb.Coder{
+				"coder1": {Spec: &pipepb.FunctionSpec{Urn: "coder1"}},
+			},
+			WindowingStrategies: map[string]*pipepb.WindowingStrategy{
+				"ws1": {WindowCoderId: "global"},
+			},
+			Environments: map[string]*pipepb.Environment{
+				"env1": {Urn: "env1"},
 			},
 		},
-		Coders: map[string]*pipepb.Coder{
-			"coder1": {Spec: &pipepb.FunctionSpec{Urn: "coder1"}},
-		},
-		WindowingStrategies: map[string]*pipepb.WindowingStrategy{
-			"ws1": {WindowCoderId: "global"},
-		},
-		Environments: map[string]*pipepb.Environment{
-			"env1": {Urn: "env1"},
-		},
-	}, []string{"e1"}
+		RemovedLeaves: []string{"e1"},
+	}
 }
diff --git a/sdks/go/pkg/beam/runners/prism/internal/separate_test.go b/sdks/go/pkg/beam/runners/prism/internal/separate_test.go
index 97ae494e4ab..1be3d3e7084 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/separate_test.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/separate_test.go
@@ -141,6 +141,7 @@ func TestSeparation(t *testing.T) {
 				count := 10
 				imp := beam.Impulse(s)
 				ints := beam.ParDo(s, emitTenFn, imp)
+				ints = beam.Reshuffle(s, ints)
 				out := beam.ParDo(s, &sepHarness{
 					Base: sepHarnessBase{
 						WatcherID:         ws.newWatcher(3),
@@ -379,7 +380,7 @@ func (fn *sepHarnessBase) setup() error {
 	sepWaitMap[fn.WatcherID] = c
 	go func(id int, c chan struct{}) {
 		for {
-			time.Sleep(time.Second * 1) // Check counts every second.
+			time.Sleep(time.Millisecond * 50) // Check counts every second.
 			sepClientMu.Lock()
 			var unblock bool
 			err := sepClient.Call("Watchers.Check", &Args{WatcherID: id}, &unblock)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/stage.go b/sdks/go/pkg/beam/runners/prism/internal/stage.go
index 4ce3ce7ffeb..4925405bb4e 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/stage.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/stage.go
@@ -128,12 +128,32 @@ func (s *stage) Execute(ctx context.Context, j *jobservices.Job, wk *worker.W, d
 	previousIndex := int64(-2)
 	var splitsDone bool
 	progTick := time.NewTicker(100 * time.Millisecond)
+	defer progTick.Stop()
+	var dataFinished, bundleFinished bool
+	// If we have no data outputs, we still need to have progress & splits
+	// while waiting for bundle completion.
+	if b.OutputCount == 0 {
+		dataFinished = true
+	}
+	var resp *fnpb.ProcessBundleResponse
 progress:
 	for {
 		select {
+		case <-ctx.Done():
+			return context.Cause(ctx)
+		case resp = <-b.Resp:
+			bundleFinished = true
+			if b.BundleErr != nil {
+				return b.BundleErr
+			}
+			if dataFinished && bundleFinished {
+				break progress // exit progress loop on close.
+			}
 		case <-dataReady:
-			progTick.Stop()
-			break progress // exit progress loop on close.
+			dataFinished = true
+			if dataFinished && bundleFinished {
+				break progress // exit progress loop on close.
+			}
 		case <-progTick.C:
 			resp, err := b.Progress(ctx, wk)
 			if err != nil {
@@ -145,9 +165,10 @@ progress:
 				md := wk.MonitoringMetadata(ctx, unknownIDs)
 				j.AddMetricShortIDs(md)
 			}
-			slog.Debug("progress report", "bundle", rb, "index", index)
+			slog.Debug("progress report", "bundle", rb, "index", index, "prevIndex", previousIndex)
 			// Progress for the bundle hasn't advanced. Try splitting.
 			if previousIndex == index && !splitsDone {
+				slog.Debug("splitting report", "bundle", rb, "index", index)
 				sr, err := b.Split(ctx, wk, 0.5 /* fraction of remainder */, nil /* allowed splits */)
 				if err != nil {
 					slog.Warn("SDK Error from split, aborting splits", "bundle", rb, "error", err.Error())
@@ -187,16 +208,6 @@ progress:
 	// Tentative Data is ready, commit it to the main datastore.
 	slog.Debug("Execute: commiting data", "bundle", rb, slog.Any("outputsWithData", maps.Keys(b.OutputData.Raw)), slog.Any("outputs", maps.Keys(s.OutputsToCoders)))
 
-	var resp *fnpb.ProcessBundleResponse
-	select {
-	case resp = <-b.Resp:
-		if b.BundleErr != nil {
-			return b.BundleErr
-		}
-	case <-ctx.Done():
-		return context.Cause(ctx)
-	}
-
 	// Tally metrics immeadiately so they're available before
 	// pipeline termination.
 	unknownIDs := j.ContributeFinalMetrics(resp)
@@ -279,6 +290,9 @@ func buildDescriptor(stg *stage, comps *pipepb.Components, wk *worker.W, ds *wor
 	for _, tid := range stg.transforms {
 		transforms[tid] = comps.GetTransforms()[tid]
 	}
+	if len(transforms) == 0 {
+		return fmt.Errorf("buildDescriptor: invalid stage - no transforms at all %v", stg.ID)
+	}
 
 	// Start with outputs, since they're simple and uniform.
 	sink2Col := map[string]string{}
@@ -287,7 +301,7 @@ func buildDescriptor(stg *stage, comps *pipepb.Components, wk *worker.W, ds *wor
 		col := comps.GetPcollections()[o.global]
 		wOutCid, err := makeWindowedValueCoder(o.global, comps, coders)
 		if err != nil {
-			return fmt.Errorf("buildDescriptor: failed to handle coder on stage %v for output %+v, pcol %q %v:\n%w", stg.ID, o, o.global, prototext.Format(col), err)
+			return fmt.Errorf("buildDescriptor: failed to handle coder on stage %v for output %+v, pcol %q %v:\n%w %v", stg.ID, o, o.global, prototext.Format(col), err, stg.transforms)
 		}
 		sinkID := o.transform + "_" + o.local
 		ed := collectionPullDecoder(col.GetCoderId(), coders, comps)
@@ -343,7 +357,7 @@ func buildDescriptor(stg *stage, comps *pipepb.Components, wk *worker.W, ds *wor
 	col := comps.GetPcollections()[stg.primaryInput]
 	wInCid, err := makeWindowedValueCoder(stg.primaryInput, comps, coders)
 	if err != nil {
-		return fmt.Errorf("buildDescriptor: failed to handle coder on stage %v for primary input, pcol %q %v:\n%w", stg.ID, stg.primaryInput, prototext.Format(col), err)
+		return fmt.Errorf("buildDescriptor: failed to handle coder on stage %v for primary input, pcol %q %v:\n%w\n%v", stg.ID, stg.primaryInput, prototext.Format(col), err, stg.transforms)
 	}
 
 	ed := collectionPullDecoder(col.GetCoderId(), coders, comps)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go b/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go
index 5f8d3875999..b8a04a7306b 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go
@@ -108,7 +108,6 @@ func TestImplemented(t *testing.T) {
 		{pipeline: primitives.Flatten},
 		{pipeline: primitives.FlattenDup},
 		{pipeline: primitives.Checkpoints},
-
 		{pipeline: primitives.CoGBK},
 		{pipeline: primitives.ReshuffleKV},
 	}