You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2024/08/12 13:35:41 UTC

(beam) branch master updated: Replace StateTag.StateBinder to top level StateBinder in SparkStateInternals (#31798)

This is an automated email from the ASF dual-hosted git repository.

kenn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 780eef98083 Replace StateTag.StateBinder to top level StateBinder in SparkStateInternals (#31798)
780eef98083 is described below

commit 780eef98083fe56f81cc5c62dc8ff193993584f0
Author: twosom <72...@users.noreply.github.com>
AuthorDate: Mon Aug 12 22:35:35 2024 +0900

    Replace StateTag.StateBinder to top level StateBinder in SparkStateInternals (#31798)
---
 ...beam_PostCommit_Java_ValidatesRunner_Spark.json |   3 +-
 ...a_ValidatesRunner_SparkStructuredStreaming.json |   3 +-
 ...stCommit_Java_ValidatesRunner_Spark_Java11.json |   3 +-
 .../org/apache/beam/runners/core/StateTag.java     |   5 +-
 .../spark/stateful/SparkStateInternals.java        | 108 ++++++++++-----------
 5 files changed, 62 insertions(+), 60 deletions(-)

diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json
index b970762c839..d59e273949d 100644
--- a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json
+++ b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json
@@ -1,4 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
+  "https://github.com/apache/beam/pull/31798": "noting that PR #31798 should run this test"
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json
index b970762c839..d59e273949d 100644
--- a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json
+++ b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json
@@ -1,4 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
+  "https://github.com/apache/beam/pull/31798": "noting that PR #31798 should run this test"
 }
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json
index b970762c839..d59e273949d 100644
--- a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json
+++ b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json
@@ -1,4 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
+  "https://github.com/apache/beam/pull/31798": "noting that PR #31798 should run this test"
 }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTag.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTag.java
index 8c699ac3111..0106f95ed74 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTag.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTag.java
@@ -69,8 +69,9 @@ public interface StateTag<StateT extends State> extends Serializable {
   /**
    * Visitor for binding a {@link StateSpec} and to the associated {@link State}.
    *
-   * @deprecated for migration only; runners should reference the top level {@link StateBinder} and
-   *     move towards {@link StateSpec} rather than {@link StateTag}.
+   * @deprecated for migration only; runners should reference the top level {@link
+   *     org.apache.beam.sdk.state.StateBinder} and move towards {@link StateSpec} rather than
+   *     {@link StateTag}.
    */
   @Deprecated
   public interface StateBinder {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
index 731cadb89f0..7ca0dc29e61 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
@@ -27,7 +27,6 @@ import java.util.function.Function;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.StateNamespace;
 import org.apache.beam.runners.core.StateTag;
-import org.apache.beam.runners.core.StateTag.StateBinder;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.InstantCoder;
@@ -42,11 +41,13 @@ import org.apache.beam.sdk.state.ReadableState;
 import org.apache.beam.sdk.state.ReadableStates;
 import org.apache.beam.sdk.state.SetState;
 import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateBinder;
 import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
-import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
+import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.CombineFnUtil;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable;
@@ -96,45 +97,47 @@ class SparkStateInternals<K> implements StateInternals {
   @Override
   public <T extends State> T state(
       StateNamespace namespace, StateTag<T> address, StateContext<?> c) {
-    return address.bind(new SparkStateBinder(namespace, c));
+    return address.getSpec().bind(address.getId(), new SparkStateBinder(namespace, c));
   }
 
   private class SparkStateBinder implements StateBinder {
     private final StateNamespace namespace;
-    private final StateContext<?> c;
+    private final StateContext<?> stateContext;
 
-    private SparkStateBinder(StateNamespace namespace, StateContext<?> c) {
+    private SparkStateBinder(StateNamespace namespace, StateContext<?> stateContext) {
       this.namespace = namespace;
-      this.c = c;
+      this.stateContext = stateContext;
     }
 
     @Override
-    public <T> ValueState<T> bindValue(StateTag<ValueState<T>> address, Coder<T> coder) {
-      return new SparkValueState<>(namespace, address, coder);
+    public <T> ValueState<T> bindValue(String id, StateSpec<ValueState<T>> spec, Coder<T> coder) {
+      return new SparkValueState<>(namespace, id, coder);
     }
 
     @Override
-    public <T> BagState<T> bindBag(StateTag<BagState<T>> address, Coder<T> elemCoder) {
-      return new SparkBagState<>(namespace, address, elemCoder);
+    public <T> BagState<T> bindBag(String id, StateSpec<BagState<T>> spec, Coder<T> elemCoder) {
+      return new SparkBagState<>(namespace, id, elemCoder);
     }
 
     @Override
-    public <T> SetState<T> bindSet(StateTag<SetState<T>> spec, Coder<T> elemCoder) {
+    public <T> SetState<T> bindSet(String id, StateSpec<SetState<T>> spec, Coder<T> elemCoder) {
       throw new UnsupportedOperationException(
           String.format("%s is not supported", SetState.class.getSimpleName()));
     }
 
     @Override
     public <KeyT, ValueT> MapState<KeyT, ValueT> bindMap(
-        StateTag<MapState<KeyT, ValueT>> address,
+        String id,
+        StateSpec<MapState<KeyT, ValueT>> spec,
         Coder<KeyT> mapKeyCoder,
         Coder<ValueT> mapValueCoder) {
-      return new SparkMapState<>(namespace, address, MapCoder.of(mapKeyCoder, mapValueCoder));
+      return new SparkMapState<>(namespace, id, MapCoder.of(mapKeyCoder, mapValueCoder));
     }
 
     @Override
     public <KeyT, ValueT> MultimapState<KeyT, ValueT> bindMultimap(
-        StateTag<MultimapState<KeyT, ValueT>> spec,
+        String id,
+        StateSpec<MultimapState<KeyT, ValueT>> spec,
         Coder<KeyT> keyCoder,
         Coder<ValueT> valueCoder) {
       throw new UnsupportedOperationException(
@@ -143,50 +146,51 @@ class SparkStateInternals<K> implements StateInternals {
 
     @Override
     public <T> OrderedListState<T> bindOrderedList(
-        StateTag<OrderedListState<T>> spec, Coder<T> elemCoder) {
+        String id, StateSpec<OrderedListState<T>> spec, Coder<T> elemCoder) {
       throw new UnsupportedOperationException(
           String.format("%s is not supported", OrderedListState.class.getSimpleName()));
     }
 
     @Override
-    public <InputT, AccumT, OutputT> CombiningState<InputT, AccumT, OutputT> bindCombiningValue(
-        StateTag<CombiningState<InputT, AccumT, OutputT>> address,
+    public <InputT, AccumT, OutputT> CombiningState<InputT, AccumT, OutputT> bindCombining(
+        String id,
+        StateSpec<CombiningState<InputT, AccumT, OutputT>> spec,
         Coder<AccumT> accumCoder,
         CombineFn<InputT, AccumT, OutputT> combineFn) {
-      return new SparkCombiningState<>(namespace, address, accumCoder, combineFn);
+      return new SparkCombiningState<>(namespace, id, accumCoder, combineFn);
     }
 
     @Override
     public <InputT, AccumT, OutputT>
-        CombiningState<InputT, AccumT, OutputT> bindCombiningValueWithContext(
-            StateTag<CombiningState<InputT, AccumT, OutputT>> address,
+        CombiningState<InputT, AccumT, OutputT> bindCombiningWithContext(
+            String id,
+            StateSpec<CombiningState<InputT, AccumT, OutputT>> spec,
             Coder<AccumT> accumCoder,
-            CombineFnWithContext<InputT, AccumT, OutputT> combineFn) {
+            CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn) {
       return new SparkCombiningState<>(
-          namespace, address, accumCoder, CombineFnUtil.bindContext(combineFn, c));
+          namespace, id, accumCoder, CombineFnUtil.bindContext(combineFn, stateContext));
     }
 
     @Override
     public WatermarkHoldState bindWatermark(
-        StateTag<WatermarkHoldState> address, TimestampCombiner timestampCombiner) {
-      return new SparkWatermarkHoldState(namespace, address, timestampCombiner);
+        String id, StateSpec<WatermarkHoldState> spec, TimestampCombiner timestampCombiner) {
+      return new SparkWatermarkHoldState(namespace, id, timestampCombiner);
     }
   }
 
   private class AbstractState<T> {
     final StateNamespace namespace;
-    final StateTag<? extends State> address;
+    final String id;
     final Coder<T> coder;
 
-    private AbstractState(
-        StateNamespace namespace, StateTag<? extends State> address, Coder<T> coder) {
+    private AbstractState(StateNamespace namespace, String id, Coder<T> coder) {
       this.namespace = namespace;
-      this.address = address;
+      this.id = id;
       this.coder = coder;
     }
 
     T readValue() {
-      byte[] buf = stateTable.get(namespace.stringKey(), address.getId());
+      byte[] buf = stateTable.get(namespace.stringKey(), id);
       if (buf != null) {
         return CoderHelpers.fromByteArray(buf, coder);
       }
@@ -194,12 +198,11 @@ class SparkStateInternals<K> implements StateInternals {
     }
 
     void writeValue(T input) {
-      stateTable.put(
-          namespace.stringKey(), address.getId(), CoderHelpers.toByteArray(input, coder));
+      stateTable.put(namespace.stringKey(), id, CoderHelpers.toByteArray(input, coder));
     }
 
     public void clear() {
-      stateTable.remove(namespace.stringKey(), address.getId());
+      stateTable.remove(namespace.stringKey(), id);
     }
 
     @Override
@@ -212,22 +215,21 @@ class SparkStateInternals<K> implements StateInternals {
       }
       @SuppressWarnings("unchecked")
       AbstractState<?> that = (AbstractState<?>) o;
-      return namespace.equals(that.namespace) && address.equals(that.address);
+      return namespace.equals(that.namespace) && id.equals(that.id);
     }
 
     @Override
     public int hashCode() {
       int result = namespace.hashCode();
-      result = 31 * result + address.hashCode();
+      result = 31 * result + id.hashCode();
       return result;
     }
   }
 
   private class SparkValueState<T> extends AbstractState<T> implements ValueState<T> {
 
-    private SparkValueState(
-        StateNamespace namespace, StateTag<ValueState<T>> address, Coder<T> coder) {
-      super(namespace, address, coder);
+    private SparkValueState(StateNamespace namespace, String id, Coder<T> coder) {
+      super(namespace, id, coder);
     }
 
     @Override
@@ -252,10 +254,8 @@ class SparkStateInternals<K> implements StateInternals {
     private final TimestampCombiner timestampCombiner;
 
     SparkWatermarkHoldState(
-        StateNamespace namespace,
-        StateTag<WatermarkHoldState> address,
-        TimestampCombiner timestampCombiner) {
-      super(namespace, address, InstantCoder.of());
+        StateNamespace namespace, String id, TimestampCombiner timestampCombiner) {
+      super(namespace, id, InstantCoder.of());
       this.timestampCombiner = timestampCombiner;
     }
 
@@ -287,7 +287,7 @@ class SparkStateInternals<K> implements StateInternals {
 
         @Override
         public Boolean read() {
-          return stateTable.get(namespace.stringKey(), address.getId()) == null;
+          return stateTable.get(namespace.stringKey(), id) == null;
         }
       };
     }
@@ -299,22 +299,22 @@ class SparkStateInternals<K> implements StateInternals {
   }
 
   @SuppressWarnings("TypeParameterShadowing")
-  private class SparkCombiningState<K, InputT, AccumT, OutputT> extends AbstractState<AccumT>
+  private class SparkCombiningState<KeyT, InputT, AccumT, OutputT> extends AbstractState<AccumT>
       implements CombiningState<InputT, AccumT, OutputT> {
 
     private final CombineFn<InputT, AccumT, OutputT> combineFn;
 
     private SparkCombiningState(
         StateNamespace namespace,
-        StateTag<CombiningState<InputT, AccumT, OutputT>> address,
+        String id,
         Coder<AccumT> coder,
         CombineFn<InputT, AccumT, OutputT> combineFn) {
-      super(namespace, address, coder);
+      super(namespace, id, coder);
       this.combineFn = combineFn;
     }
 
     @Override
-    public SparkCombiningState<K, InputT, AccumT, OutputT> readLater() {
+    public SparkCombiningState<KeyT, InputT, AccumT, OutputT> readLater() {
       return this;
     }
 
@@ -348,7 +348,7 @@ class SparkStateInternals<K> implements StateInternals {
 
         @Override
         public Boolean read() {
-          return stateTable.get(namespace.stringKey(), address.getId()) == null;
+          return stateTable.get(namespace.stringKey(), id) == null;
         }
       };
     }
@@ -369,10 +369,8 @@ class SparkStateInternals<K> implements StateInternals {
       extends AbstractState<Map<MapKeyT, MapValueT>> implements MapState<MapKeyT, MapValueT> {
 
     private SparkMapState(
-        StateNamespace namespace,
-        StateTag<? extends State> address,
-        Coder<Map<MapKeyT, MapValueT>> coder) {
-      super(namespace, address, coder);
+        StateNamespace namespace, String id, Coder<Map<MapKeyT, MapValueT>> coder) {
+      super(namespace, id, coder);
     }
 
     @Override
@@ -490,7 +488,7 @@ class SparkStateInternals<K> implements StateInternals {
       return new ReadableState<Boolean>() {
         @Override
         public Boolean read() {
-          return stateTable.get(namespace.stringKey(), address.getId()) == null;
+          return stateTable.get(namespace.stringKey(), id) == null;
         }
 
         @Override
@@ -502,8 +500,8 @@ class SparkStateInternals<K> implements StateInternals {
   }
 
   private final class SparkBagState<T> extends AbstractState<List<T>> implements BagState<T> {
-    private SparkBagState(StateNamespace namespace, StateTag<BagState<T>> address, Coder<T> coder) {
-      super(namespace, address, ListCoder.of(coder));
+    private SparkBagState(StateNamespace namespace, String id, Coder<T> coder) {
+      super(namespace, id, ListCoder.of(coder));
     }
 
     @Override
@@ -537,7 +535,7 @@ class SparkStateInternals<K> implements StateInternals {
 
         @Override
         public Boolean read() {
-          return stateTable.get(namespace.stringKey(), address.getId()) == null;
+          return stateTable.get(namespace.stringKey(), id) == null;
         }
       };
     }