You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2025/01/09 00:34:19 UTC
(beam) branch master updated: [#31438] WindowingStrategy Plumbing + AllowedLateness + Fixing Session Windows (#33542)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4fc5c869843 [#31438] WindowingStrategy Plumbing + AllowedLateness + Fixing Session Windows (#33542)
4fc5c869843 is described below
commit 4fc5c869843e9439b3c73f80480d9b9ef651858a
Author: Robert Burke <lo...@users.noreply.github.com>
AuthorDate: Wed Jan 8 16:34:11 2025 -0800
[#31438] WindowingStrategy Plumbing + AllowedLateness + Fixing Session Windows (#33542)
---
runners/prism/java/build.gradle | 15 ---
.../prism/internal/engine/elementmanager.go | 44 ++++---
.../beam/runners/prism/internal/engine/strategy.go | 28 ++---
.../runners/prism/internal/engine/strategy_test.go | 13 +-
sdks/go/pkg/beam/runners/prism/internal/execute.go | 9 +-
.../beam/runners/prism/internal/handlerunner.go | 133 +++++++++++----------
.../prism/internal/jobservices/management.go | 5 +-
.../pkg/beam/runners/prism/internal/urns/urns.go | 26 ++--
.../runners/portability/prism_runner_test.py | 23 ----
9 files changed, 129 insertions(+), 167 deletions(-)
diff --git a/runners/prism/java/build.gradle b/runners/prism/java/build.gradle
index 791952c625f..3aae54c4a8c 100644
--- a/runners/prism/java/build.gradle
+++ b/runners/prism/java/build.gradle
@@ -96,9 +96,6 @@ def sickbayTests = [
'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testAfterProcessingTimeContinuationTriggerEarly',
'org.apache.beam.sdk.transforms.ParDoTest$BundleInvariantsTests.testWatermarkUpdateMidBundle',
'org.apache.beam.sdk.transforms.ViewTest.testTriggeredLatestSingleton',
- // Requires Allowed Lateness, among others.
- 'org.apache.beam.sdk.transforms.ParDoTest$TimerTests.testEventTimeTimerSetWithinAllowedLateness',
- 'org.apache.beam.sdk.transforms.ParDoTest$StateTests.testRequiresTimeSortedInputWithLateDataAndAllowedLateness',
'org.apache.beam.sdk.testing.TestStreamTest.testFirstElementLate',
'org.apache.beam.sdk.testing.TestStreamTest.testDiscardingMode',
'org.apache.beam.sdk.testing.TestStreamTest.testEarlyPanesOfWindow',
@@ -116,10 +113,6 @@ def sickbayTests = [
// Coding error somehow: short write: reached end of stream after reading 5 bytes; 98 bytes expected
'org.apache.beam.sdk.testing.TestStreamTest.testMultiStage',
- // Prism not firing sessions correctly (seems to be merging inapppropriately)
- 'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testSessionsCombine',
- 'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testSessionsCombineWithContext',
-
// Java side dying during execution.
// https://github.com/apache/beam/issues/32930
'org.apache.beam.sdk.transforms.FlattenTest.testFlattenMultipleCoders',
@@ -161,14 +154,6 @@ def sickbayTests = [
// TODO(https://github.com/apache/beam/issues/31231)
'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributePreservesMetadata',
-
- // These tests fail once Late Data was being precisely dropped.
- // They set a single element to be late data, and expect it (correctly) to be preserved.
- // Since presently, these are treated as No-ops, the fix is to disable the
- // dropping behavior when a stage's input is a Reshuffle/Redistribute transform.
- 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshuffleWithTimestampsStreaming',
- 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributeWithTimestampsStreaming',
-
// Prism isn't handling Java's side input views properly.
// https://github.com/apache/beam/issues/32932
// java.lang.IllegalArgumentException: PCollection with more than one element accessed as a singleton view.
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
index bb3c8ceceeb..fb1ae47d301 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
@@ -261,8 +261,10 @@ func (em *ElementManager) AddStage(ID string, inputIDs, outputIDs []string, side
// StageAggregates marks the given stage as an aggregation, which
// means elements will only be processed based on windowing strategies.
-func (em *ElementManager) StageAggregates(ID string) {
- em.stages[ID].aggregate = true
+func (em *ElementManager) StageAggregates(ID string, strat WinStrat) {
+ ss := em.stages[ID]
+ ss.aggregate = true
+ ss.strat = strat
}
// StageStateful marks the given stage as stateful, which means elements are
@@ -1095,7 +1097,7 @@ type stageState struct {
// Special handling bits
stateful bool // whether this stage uses state or timers, and needs keyed processing.
aggregate bool // whether this stage needs to block for aggregation.
- strat winStrat // Windowing Strategy for aggregation fireings.
+ strat WinStrat // Windowing Strategy for aggregation fireings.
processingTimeTimersFamilies map[string]bool // Indicates which timer families use the processing time domain.
// onWindowExpiration management
@@ -1154,7 +1156,6 @@ func makeStageState(ID string, inputIDs, outputIDs []string, sides []LinkID) *st
ID: ID,
outputIDs: outputIDs,
sides: sides,
- strat: defaultStrat{},
state: map[LinkID]map[typex.Window]map[string]StateData{},
watermarkHolds: newHoldTracker(),
@@ -1179,18 +1180,21 @@ func makeStageState(ID string, inputIDs, outputIDs []string, sides []LinkID) *st
func (ss *stageState) AddPending(newPending []element) int {
ss.mu.Lock()
defer ss.mu.Unlock()
- // TODO(#https://github.com/apache/beam/issues/31438):
- // Adjust with AllowedLateness
- // Data that arrives after the *output* watermark is late.
- threshold := ss.output
- origPending := make([]element, 0, ss.pending.Len())
- for _, e := range newPending {
- if e.window.MaxTimestamp() < threshold {
- continue
+ if ss.aggregate {
+ // Late Data is data that has arrived after that window has expired.
+ // We only need to drop late data before aggregations.
+ // TODO - handle for side inputs too.
+ threshold := ss.output
+ origPending := make([]element, 0, ss.pending.Len())
+ for _, e := range newPending {
+ if ss.strat.EarliestCompletion(e.window) < threshold {
+ // TODO: figure out Pane and trigger firings.
+ continue
+ }
+ origPending = append(origPending, e)
}
- origPending = append(origPending, e)
+ newPending = origPending
}
- newPending = origPending
if ss.stateful {
if ss.pendingByKeys == nil {
ss.pendingByKeys = map[string]*dataAndTimers{}
@@ -1626,10 +1630,8 @@ func (ss *stageState) updateWatermarks(em *ElementManager) set[string] {
// They'll never be read in again.
for _, wins := range ss.sideInputs {
for win := range wins {
- // TODO(#https://github.com/apache/beam/issues/31438):
- // Adjust with AllowedLateness
// Clear out anything we've already used.
- if win.MaxTimestamp() < newOut {
+ if ss.strat.EarliestCompletion(win) < newOut {
// If the expiry is in progress, skip this window.
if ss.inProgressExpiredWindows[win] > 0 {
continue
@@ -1640,9 +1642,7 @@ func (ss *stageState) updateWatermarks(em *ElementManager) set[string] {
}
for _, wins := range ss.state {
for win := range wins {
- // TODO(#https://github.com/apache/beam/issues/31438):
- // Adjust with AllowedLateness
- if win.MaxTimestamp() < newOut {
+ if ss.strat.EarliestCompletion(win) < newOut {
// If the expiry is in progress, skip collecting this window.
if ss.inProgressExpiredWindows[win] > 0 {
continue
@@ -1685,9 +1685,7 @@ func (ss *stageState) createOnWindowExpirationBundles(newOut mtime.Time, em *Ele
var preventDownstreamUpdate bool
for win, keys := range ss.keysToExpireByWindow {
// Check if the window has expired.
- // TODO(#https://github.com/apache/beam/issues/31438):
- // Adjust with AllowedLateness
- if win.MaxTimestamp() >= newOut {
+ if ss.strat.EarliestCompletion(win) >= newOut {
continue
}
// We can't advance the output watermark if there's garbage to collect.
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/strategy.go b/sdks/go/pkg/beam/runners/prism/internal/engine/strategy.go
index 44e6064958c..b0af2a09a75 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/strategy.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/strategy.go
@@ -23,28 +23,16 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
)
-type winStrat interface {
- EarliestCompletion(typex.Window) mtime.Time
+// WinStrat configures the windowing strategy for the stage, based on the
+// stage's input PCollection.
+type WinStrat struct {
+ AllowedLateness time.Duration // Used to extend duration
}
-type defaultStrat struct{}
-
-func (ws defaultStrat) EarliestCompletion(w typex.Window) mtime.Time {
- return w.MaxTimestamp()
-}
-
-func (defaultStrat) String() string {
- return "default"
-}
-
-type sessionStrat struct {
- GapSize time.Duration
-}
-
-func (ws sessionStrat) EarliestCompletion(w typex.Window) mtime.Time {
- return w.MaxTimestamp().Add(ws.GapSize)
+func (ws WinStrat) EarliestCompletion(w typex.Window) mtime.Time {
+ return w.MaxTimestamp().Add(ws.AllowedLateness)
}
-func (ws sessionStrat) String() string {
- return fmt.Sprintf("session[GapSize:%v]", ws.GapSize)
+func (ws WinStrat) String() string {
+ return fmt.Sprintf("WinStrat[AllowedLateness:%v]", ws.AllowedLateness)
}
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/strategy_test.go b/sdks/go/pkg/beam/runners/prism/internal/engine/strategy_test.go
index 9d558396f80..845f748064c 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/strategy_test.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/strategy_test.go
@@ -26,15 +26,16 @@ import (
func TestEarliestCompletion(t *testing.T) {
tests := []struct {
- strat winStrat
+ strat WinStrat
input typex.Window
want mtime.Time
}{
- {defaultStrat{}, window.GlobalWindow{}, mtime.EndOfGlobalWindowTime},
- {defaultStrat{}, window.IntervalWindow{Start: 0, End: 4}, 3},
- {defaultStrat{}, window.IntervalWindow{Start: mtime.MinTimestamp, End: mtime.MaxTimestamp}, mtime.MaxTimestamp - 1},
- {sessionStrat{}, window.IntervalWindow{Start: 0, End: 4}, 3},
- {sessionStrat{GapSize: 3 * time.Millisecond}, window.IntervalWindow{Start: 0, End: 4}, 6},
+ {WinStrat{}, window.GlobalWindow{}, mtime.EndOfGlobalWindowTime},
+ {WinStrat{}, window.IntervalWindow{Start: 0, End: 4}, 3},
+ {WinStrat{}, window.IntervalWindow{Start: mtime.MinTimestamp, End: mtime.MaxTimestamp}, mtime.MaxTimestamp - 1},
+ {WinStrat{AllowedLateness: 5 * time.Second}, window.GlobalWindow{}, mtime.EndOfGlobalWindowTime.Add(5 * time.Second)},
+ {WinStrat{AllowedLateness: 5 * time.Millisecond}, window.IntervalWindow{Start: 0, End: 4}, 8},
+ {WinStrat{AllowedLateness: 5 * time.Second}, window.IntervalWindow{Start: mtime.MinTimestamp, End: mtime.MaxTimestamp}, mtime.MaxTimestamp.Add(5 * time.Second)},
}
for _, test := range tests {
diff --git a/sdks/go/pkg/beam/runners/prism/internal/execute.go b/sdks/go/pkg/beam/runners/prism/internal/execute.go
index d41c3cd9c75..ecc9cecdaa5 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/execute.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/execute.go
@@ -223,7 +223,10 @@ func executePipeline(ctx context.Context, wks map[string]*worker.W, j *jobservic
KeyDec: kd,
}
}
- em.StageAggregates(stage.ID)
+ ws := windowingStrategy(comps, tid)
+ em.StageAggregates(stage.ID, engine.WinStrat{
+ AllowedLateness: time.Duration(ws.GetAllowedLateness()) * time.Millisecond,
+ })
case urns.TransformImpulse:
impulses = append(impulses, stage.ID)
em.AddStage(stage.ID, nil, []string{getOnlyValue(t.GetOutputs())}, nil)
@@ -266,11 +269,11 @@ func executePipeline(ctx context.Context, wks map[string]*worker.W, j *jobservic
case *pipepb.TestStreamPayload_Event_ElementEvent:
var elms []engine.TestStreamElement
for _, e := range ev.ElementEvent.GetElements() {
- elms = append(elms, engine.TestStreamElement{Encoded: mayLP(e.GetEncodedElement()), EventTime: mtime.Time(e.GetTimestamp())})
+ elms = append(elms, engine.TestStreamElement{Encoded: mayLP(e.GetEncodedElement()), EventTime: mtime.FromMilliseconds(e.GetTimestamp())})
}
tsb.AddElementEvent(ev.ElementEvent.GetTag(), elms)
case *pipepb.TestStreamPayload_Event_WatermarkEvent:
- tsb.AddWatermarkEvent(ev.WatermarkEvent.GetTag(), mtime.Time(ev.WatermarkEvent.GetNewWatermark()))
+ tsb.AddWatermarkEvent(ev.WatermarkEvent.GetTag(), mtime.FromMilliseconds(ev.WatermarkEvent.GetNewWatermark()))
case *pipepb.TestStreamPayload_Event_ProcessingTimeEvent:
if ev.ProcessingTimeEvent.GetAdvanceDuration() == int64(mtime.MaxTimestamp) {
// TODO: Determine the SDK common formalism for setting processing time to infinity.
diff --git a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
index be9d39ad02b..340eb0b7eb5 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
@@ -19,7 +19,6 @@ import (
"bytes"
"fmt"
"io"
- "log/slog"
"reflect"
"sort"
@@ -33,7 +32,6 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns"
"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/worker"
"google.golang.org/protobuf/encoding/prototext"
- "google.golang.org/protobuf/proto"
)
// This file retains the logic for the pardo handler
@@ -67,7 +65,12 @@ func (*runner) ConfigCharacteristic() reflect.Type {
var _ transformPreparer = (*runner)(nil)
func (*runner) PrepareUrns() []string {
- return []string{urns.TransformReshuffle, urns.TransformFlatten}
+ return []string{
+ urns.TransformReshuffle,
+ urns.TransformRedistributeArbitrarily,
+ urns.TransformRedistributeByKey,
+ urns.TransformFlatten,
+ }
}
// PrepareTransform handles special processing with respect runner transforms, like reshuffle.
@@ -75,7 +78,7 @@ func (h *runner) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipep
switch t.GetSpec().GetUrn() {
case urns.TransformFlatten:
return h.handleFlatten(tid, t, comps)
- case urns.TransformReshuffle:
+ case urns.TransformReshuffle, urns.TransformRedistributeArbitrarily, urns.TransformRedistributeByKey:
return h.handleReshuffle(tid, t, comps)
default:
panic("unknown urn to Prepare: " + t.GetSpec().GetUrn())
@@ -190,7 +193,13 @@ func (h *runner) handleReshuffle(tid string, t *pipepb.PTransform, comps *pipepb
var _ transformExecuter = (*runner)(nil)
func (*runner) ExecuteUrns() []string {
- return []string{urns.TransformFlatten, urns.TransformGBK, urns.TransformReshuffle}
+ return []string{
+ urns.TransformFlatten,
+ urns.TransformGBK,
+ urns.TransformReshuffle,
+ urns.TransformRedistributeArbitrarily,
+ urns.TransformRedistributeByKey,
+ }
}
// ExecuteWith returns what environment the transform should execute in.
@@ -289,6 +298,17 @@ func windowingStrategy(comps *pipepb.Components, tid string) *pipepb.WindowingSt
return comps.GetWindowingStrategies()[pcol.GetWindowingStrategyId()]
}
+// getOrMake is a generic helper function for extracting or initializing a sub map.
+// Avoids an amount of boiler plate.
+func getOrMake[K, VK comparable, VV any, V map[VK]VV, M map[K]V](m M, key K) V {
+ v, ok := m[key]
+ if !ok {
+ v = make(V)
+ m[key] = v
+ }
+ return v
+}
+
// gbkBytes re-encodes gbk inputs in a gbk result.
func gbkBytes(ws *pipepb.WindowingStrategy, wc, kc, vc *pipepb.Coder, toAggregate [][]byte, coders map[string]*pipepb.Coder) []byte {
// Pick how the timestamp of the aggregated output is computed.
@@ -325,15 +345,15 @@ func gbkBytes(ws *pipepb.WindowingStrategy, wc, kc, vc *pipepb.Coder, toAggregat
time mtime.Time
values [][]byte
}
- // Map windows to a map of keys to a map of keys to time.
+ // Map keys to windows to element batches.
// We ultimately emit the window, the key, the time, and the iterable of elements,
// all contained in the final value.
- windows := map[typex.Window]map[string]keyTime{}
+ keys := map[string]map[typex.Window]keyTime{}
kd := pullDecoder(kc, coders)
vd := pullDecoder(vc, coders)
- // Aggregate by windows and keys, using the window coder and KV coders.
+ // Aggregate by keys, and windows, using the window coder and KV coders.
// We need to extract and split the key bytes from the element bytes.
for _, data := range toAggregate {
// Parse out each element's data, and repeat.
@@ -351,12 +371,8 @@ func gbkBytes(ws *pipepb.WindowingStrategy, wc, kc, vc *pipepb.Coder, toAggregat
key := string(keyByt)
value := vd(buf)
for _, w := range ws {
- wk, ok := windows[w]
- if !ok {
- wk = make(map[string]keyTime)
- windows[w] = wk
- }
- kt, ok := wk[key]
+ wins := getOrMake(keys, key)
+ kt, ok := wins[w]
if !ok {
// If the window+key map doesn't have a value, inititialize time with the element time.
// This allows earliest or latest to work properly in the outputTime function's first use.
@@ -366,69 +382,64 @@ func gbkBytes(ws *pipepb.WindowingStrategy, wc, kc, vc *pipepb.Coder, toAggregat
kt.key = keyByt
kt.w = w
kt.values = append(kt.values, value)
- wk[key] = kt
+ wins[w] = kt
}
}
}
// If the strategy is session windows, then we need to get all the windows, sort them
// and see which ones need to be merged together.
+ // Each key has their windows merged separately.
if ws.GetWindowFn().GetUrn() == urns.WindowFnSession {
- slog.Debug("sorting by session window")
- session := &pipepb.SessionWindowsPayload{}
- if err := (proto.UnmarshalOptions{}).Unmarshal(ws.GetWindowFn().GetPayload(), session); err != nil {
- panic("unable to decode SessionWindowsPayload")
- }
- gapSize := mtime.Time(session.GetGapSize().AsDuration())
-
- ordered := make([]window.IntervalWindow, 0, len(windows))
- for k := range windows {
- ordered = append(ordered, k.(window.IntervalWindow))
- }
- // Use a decreasing sort (latest to earliest) so we can correct
- // the output timestamp to the new end of window immeadiately.
- sort.Slice(ordered, func(i, j int) bool {
- return ordered[i].MaxTimestamp() > ordered[j].MaxTimestamp()
- })
-
- cur := ordered[0]
- sessionData := windows[cur]
- delete(windows, cur)
- for _, iw := range ordered[1:] {
- // Check if the gap between windows is less than the gapSize.
- // If not, this window is done, and we start a next window.
- if iw.End+gapSize < cur.Start {
- // Store current data with the current window.
- windows[cur] = sessionData
- // Use the incoming window instead, and clear it from the map.
- cur = iw
- sessionData = windows[iw]
- delete(windows, cur)
- // There's nothing to merge, since we've just started with this windowed data.
- continue
+ for _, windows := range keys {
+ ordered := make([]window.IntervalWindow, 0, len(windows))
+ for win := range windows {
+ ordered = append(ordered, win.(window.IntervalWindow))
}
- // Extend the session with the incoming window, and merge the the incoming window's data.
- cur.Start = iw.Start
- toMerge := windows[iw]
- delete(windows, iw)
- for k, kt := range toMerge {
- skt := sessionData[k]
+ // Use a decreasing sort (latest to earliest) so we can correct
+ // the output timestamp to the new end of window immeadiately.
+ sort.Slice(ordered, func(i, j int) bool {
+ return ordered[i].MaxTimestamp() > ordered[j].MaxTimestamp()
+ })
+
+ cur := ordered[0]
+ sessionData := windows[cur]
+ delete(windows, cur)
+ for _, iw := range ordered[1:] {
+ // GapSize is already incorporated into the windows,
+ // check for consecutive windows that don't overlap.
+ if cur.Start-iw.End > 0 {
+ // If so, this window is done, and we start a next window.
+ // Store current data with the current window.
+ windows[cur] = sessionData
+ // Use the incoming window instead, and clear it from the map.
+ cur = iw
+ sessionData = windows[iw]
+ delete(windows, cur)
+ // There's nothing to merge, since we've just started with this windowed data.
+ continue
+ }
+ // Extend the session with the incoming window, and merge the the incoming window's data.
+ cur.Start = iw.Start
+ toMerge := windows[iw]
+ delete(windows, iw)
+
// Ensure the output time matches the given function.
- skt.time = outputTime(cur, kt.time, skt.time)
- skt.key = kt.key
- skt.w = cur
- skt.values = append(skt.values, kt.values...)
- sessionData[k] = skt
+ sessionData.time = outputTime(cur, toMerge.time, sessionData.time)
+ sessionData.key = toMerge.key
+ sessionData.w = cur
+ // TODO: May need to adjust the ordering here.
+ sessionData.values = append(sessionData.values, toMerge.values...)
}
+ windows[cur] = sessionData
}
- windows[cur] = sessionData
}
// Everything's aggregated!
// Time to turn things into a windowed KV<K, Iterable<V>>
var buf bytes.Buffer
- for _, w := range windows {
- for _, kt := range w {
+ for _, wins := range keys {
+ for _, kt := range wins {
exec.EncodeWindowedValueHeader(
wEnc,
[]typex.Window{kt.w},
diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
index b9a28e4bc65..a4307b706fa 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
@@ -23,7 +23,6 @@ import (
"sync"
"sync/atomic"
- "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
jobpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagement_v1"
pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1"
"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns"
@@ -144,7 +143,7 @@ func (s *Server) Prepare(ctx context.Context, req *jobpb.PrepareJobRequest) (_ *
urns.TransformAssignWindows:
// Very few expected transforms types for submitted pipelines.
// Most URNs are for the runner to communicate back to the SDK for execution.
- case urns.TransformReshuffle:
+ case urns.TransformReshuffle, urns.TransformRedistributeArbitrarily, urns.TransformRedistributeByKey:
// Reshuffles use features we don't yet support, but we would like to
// support them by making them the no-op they are, and be precise about
// what we're ignoring.
@@ -227,8 +226,6 @@ func (s *Server) Prepare(ctx context.Context, req *jobpb.PrepareJobRequest) (_ *
// Inspect Windowing strategies for unsupported features.
for wsID, ws := range job.Pipeline.GetComponents().GetWindowingStrategies() {
- check("WindowingStrategy.AllowedLateness", ws.GetAllowedLateness(), int64(0), mtime.MaxTimestamp.Milliseconds())
-
// Both Closing behaviors are identical without additional trigger firings.
check("WindowingStrategy.ClosingBehaviour", ws.GetClosingBehavior(), pipepb.ClosingBehavior_EMIT_IF_NONEMPTY, pipepb.ClosingBehavior_EMIT_ALWAYS)
check("WindowingStrategy.AccumulationMode", ws.GetAccumulationMode(), pipepb.AccumulationMode_DISCARDING)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go b/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go
index 12e62ef84a8..170073b7241 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go
@@ -56,18 +56,20 @@ var (
var (
// SDK transforms.
- TransformParDo = ptUrn(pipepb.StandardPTransforms_PAR_DO)
- TransformCombinePerKey = ctUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY)
- TransformCombineGlobally = ctUrn(pipepb.StandardPTransforms_COMBINE_GLOBALLY)
- TransformReshuffle = ctUrn(pipepb.StandardPTransforms_RESHUFFLE)
- TransformCombineGroupedValues = cmbtUrn(pipepb.StandardPTransforms_COMBINE_GROUPED_VALUES)
- TransformPreCombine = cmbtUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY_PRECOMBINE)
- TransformMerge = cmbtUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY_MERGE_ACCUMULATORS)
- TransformExtract = cmbtUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY_EXTRACT_OUTPUTS)
- TransformPairWithRestriction = sdfUrn(pipepb.StandardPTransforms_PAIR_WITH_RESTRICTION)
- TransformSplitAndSize = sdfUrn(pipepb.StandardPTransforms_SPLIT_AND_SIZE_RESTRICTIONS)
- TransformProcessSizedElements = sdfUrn(pipepb.StandardPTransforms_PROCESS_SIZED_ELEMENTS_AND_RESTRICTIONS)
- TransformTruncate = sdfUrn(pipepb.StandardPTransforms_TRUNCATE_SIZED_RESTRICTION)
+ TransformParDo = ptUrn(pipepb.StandardPTransforms_PAR_DO)
+ TransformCombinePerKey = ctUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY)
+ TransformCombineGlobally = ctUrn(pipepb.StandardPTransforms_COMBINE_GLOBALLY)
+ TransformReshuffle = ctUrn(pipepb.StandardPTransforms_RESHUFFLE)
+ TransformRedistributeArbitrarily = ctUrn(pipepb.StandardPTransforms_REDISTRIBUTE_ARBITRARILY)
+ TransformRedistributeByKey = ctUrn(pipepb.StandardPTransforms_REDISTRIBUTE_BY_KEY)
+ TransformCombineGroupedValues = cmbtUrn(pipepb.StandardPTransforms_COMBINE_GROUPED_VALUES)
+ TransformPreCombine = cmbtUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY_PRECOMBINE)
+ TransformMerge = cmbtUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY_MERGE_ACCUMULATORS)
+ TransformExtract = cmbtUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY_EXTRACT_OUTPUTS)
+ TransformPairWithRestriction = sdfUrn(pipepb.StandardPTransforms_PAIR_WITH_RESTRICTION)
+ TransformSplitAndSize = sdfUrn(pipepb.StandardPTransforms_SPLIT_AND_SIZE_RESTRICTIONS)
+ TransformProcessSizedElements = sdfUrn(pipepb.StandardPTransforms_PROCESS_SIZED_ELEMENTS_AND_RESTRICTIONS)
+ TransformTruncate = sdfUrn(pipepb.StandardPTransforms_TRUNCATE_SIZED_RESTRICTION)
// Window Manipulation
TransformAssignWindows = ptUrn(pipepb.StandardPTransforms_ASSIGN_WINDOWS)
diff --git a/sdks/python/apache_beam/runners/portability/prism_runner_test.py b/sdks/python/apache_beam/runners/portability/prism_runner_test.py
index 337ac991948..1e8880a60a3 100644
--- a/sdks/python/apache_beam/runners/portability/prism_runner_test.py
+++ b/sdks/python/apache_beam/runners/portability/prism_runner_test.py
@@ -34,9 +34,6 @@ from apache_beam.options.pipeline_options import DebugOptions
from apache_beam.options.pipeline_options import PortableOptions
from apache_beam.runners.portability import portable_runner_test
from apache_beam.testing.util import assert_that
-from apache_beam.testing.util import equal_to
-from apache_beam.transforms import window
-from apache_beam.utils import timestamp
# Run as
#
@@ -174,26 +171,6 @@ class PrismRunnerTest(portable_runner_test.PortableRunnerTest):
return options
- # Slightly more robust session window test:
- # Validates that an inner grouping doesn't duplicate data either.
- # Copied also because the timestamp in fn_runner_test.py isn't being
- # inferred correctly as seconds for some reason, but as micros.
- # The belabored specification is validating the timestamp type works at least.
- # See https://github.com/apache/beam/issues/32085
- def test_windowing(self):
- with self.create_pipeline() as p:
- res = (
- p
- | beam.Create([1, 2, 100, 101, 102, 123])
- | beam.Map(
- lambda t: window.TimestampedValue(
- ('k', t), timestamp.Timestamp.of(t).micros))
- | beam.WindowInto(beam.transforms.window.Sessions(10))
- | beam.GroupByKey()
- | beam.Map(lambda k_vs1: (k_vs1[0], sorted(k_vs1[1]))))
- assert_that(
- res, equal_to([('k', [1, 2]), ('k', [100, 101, 102]), ('k', [123])]))
-
# Can't read host files from within docker, read a "local" file there.
def test_read(self):
print('name:', __name__)