You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ie...@apache.org on 2019/04/05 14:54:24 UTC

[beam] branch master updated: [BEAM-6964] Simplify key context handling in ExecutableStageDoFnOperator

This is an automated email from the ASF dual-hosted git repository.

iemejia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 4e39c01  [BEAM-6964] Simplify key context handling in ExecutableStageDoFnOperator
     new 4100fb9  Merge pull request #8190: [BEAM-6964] Simplify key context handling in ExecutableStageDoFnOperator
4e39c01 is described below

commit 4e39c01c2319a06142bd7d578e90a82537463716
Author: Maximilian Michels <mx...@apache.org>
AuthorDate: Mon Apr 1 14:53:28 2019 +0200

    [BEAM-6964] Simplify key context handling in ExecutableStageDoFnOperator
    
    The key handling code in ExecutableStageDoFnOperator was a bit hard to
    understand because of the different contexts in which the current key is
    set. This simplifies the key setting by always using the state backend to set
    the key. An additional guard has been added to ensure that no concurrent access
    is performed during key access.
    
    This also adds an additional class for consistent use of key coders.
---
 .../streaming/ExecutableStageDoFnOperator.java     | 99 +++++++---------------
 .../wrappers/streaming/FlinkKeyUtils.java          | 63 ++++++++++++++
 .../streaming/KvToByteBufferKeySelector.java       |  4 +-
 .../wrappers/streaming/WorkItemKeySelector.java    |  4 +-
 .../streaming/ExecutableStageDoFnOperatorTest.java |  7 +-
 .../wrappers/streaming/FlinkKeyUtilsTest.java      | 63 ++++++++++++++
 6 files changed, 159 insertions(+), 81 deletions(-)

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperator.java
index dcc473f..bb26daa 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperator.java
@@ -19,7 +19,6 @@ package org.apache.beam.runners.flink.translation.wrappers.streaming;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
@@ -35,7 +34,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.BiConsumer;
-import java.util.function.Consumer;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleProgressResponse;
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleResponse;
@@ -110,7 +109,7 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
   private final Map<String, TupleTag<?>> outputMap;
   private final Map<RunnerApi.ExecutableStagePayload.SideInputId, PCollectionView<?>> sideInputIds;
   /** A lock which has to be acquired when concurrently accessing state and timers. */
-  private final Lock stateBackendLock;
+  private final ReentrantLock stateBackendLock;
 
   private transient FlinkExecutableStageContext stageContext;
   private transient StateRequestHandler stateRequestHandler;
@@ -331,13 +330,16 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
         }
 
         private void prepareStateBackend(K key, Coder<K> keyCoder) {
-          ByteArrayOutputStream baos = new ByteArrayOutputStream();
+          final ByteBuffer encodedKey;
           try {
-            keyCoder.encode(key, baos);
-          } catch (IOException e) {
-            throw new RuntimeException("Failed to encode key for Flink state backend", e);
+            // We need to have NESTED context here with the ByteStringCoder.
+            // See StateRequestHandlers.
+            encodedKey =
+                ByteBuffer.wrap(CoderUtils.encodeToByteArray(keyCoder, key, Coder.Context.NESTED));
+          } catch (CoderException e) {
+            throw new RuntimeException("Couldn't set key for state");
           }
-          keyedStateBackend.setCurrentKey(ByteBuffer.wrap(baos.toByteArray()));
+          keyedStateBackend.setCurrentKey(encodedKey);
         }
       };
     }
@@ -365,47 +367,36 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
   public void setCurrentKey(Object key) {}
 
   @Override
-  public Object getCurrentKey() {
-    // This is the key retrieved by HeapInternalTimerService when setting a Flink timer
-    return sdkHarnessRunner.getCurrentTimerKey();
+  public ByteBuffer getCurrentKey() {
+    // This is the key retrieved by HeapInternalTimerService when setting a Flink timer.
+    // Note: Only called by the TimerService. Must be guarded by a lock.
+    Preconditions.checkState(
+        stateBackendLock.isLocked(),
+        "State backend must be locked when retrieving the current key.");
+    return this.<ByteBuffer>getKeyedStateBackend().getCurrentKey();
   }
 
   private void setTimer(WindowedValue<InputT> timerElement, TimerInternals.TimerData timerData) {
     try {
-      Object key = keySelector.getKey(timerElement);
-      sdkHarnessRunner.setCurrentTimerKey(key);
+      // KvToByteBufferKeySelector returns the key encoded
+      ByteBuffer encodedKey = (ByteBuffer) keySelector.getKey(timerElement);
       // We have to synchronize to ensure the state backend is not concurrently accessed by the
       // state requests
       try {
         stateBackendLock.lock();
-        getKeyedStateBackend().setCurrentKey(key);
+        getKeyedStateBackend().setCurrentKey(encodedKey);
         timerInternals.setTimer(timerData);
       } finally {
         stateBackendLock.unlock();
       }
     } catch (Exception e) {
       throw new RuntimeException("Couldn't set timer", e);
-    } finally {
-      sdkHarnessRunner.setCurrentTimerKey(null);
     }
   }
 
   @Override
   public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) {
-    // We need to decode the key
     final ByteBuffer encodedKey = (ByteBuffer) timer.getKey();
-    @SuppressWarnings("ByteBufferBackingArray")
-    byte[] bytes = encodedKey.array();
-    final Object decodedKey;
-    try {
-      decodedKey = CoderUtils.decodeFromByteArray(keyCoder, bytes);
-    } catch (CoderException e) {
-      throw new RuntimeException(
-          String.format(Locale.ENGLISH, "Failed to decode encoded key: %s", Arrays.toString(bytes)),
-          e);
-    }
-    // Prepare the SdkHarnessRunner with the key for the timer
-    sdkHarnessRunner.setCurrentTimerKey(decodedKey);
     // We have to synchronize to ensure the state backend is not concurrently accessed by the state
     // requests
     try {
@@ -455,8 +446,8 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
             outputManager,
             outputMap,
             (Coder<BoundedWindow>) windowingStrategy.getWindowFn().windowCoder(),
-            keySelector,
-            this::setTimer);
+            this::setTimer,
+            () -> FlinkKeyUtils.decodeKey(getCurrentKey(), keyCoder));
 
     return ensureStateCleanup(sdkHarnessRunner);
   }
@@ -529,16 +520,11 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
     private final Map<String, TimerSpec> timerOutputIdToSpecMap;
 
     private final Coder<BoundedWindow> windowCoder;
-    private final KeySelector<WindowedValue<InputT>, ?> keySelector;
     private final BiConsumer<WindowedValue<InputT>, TimerInternals.TimerData> timerRegistration;
+    private final Supplier<Object> keyForTimer;
 
     private RemoteBundle remoteBundle;
     private FnDataReceiver<WindowedValue<?>> mainInputReceiver;
-    // Timer key set before calling Flink's internal timer service to register
-    // a timer. The timer service will retrieve this with a call to {@code getCurrentKey}.
-    // Before firing a timer, this will be initialized with the current key
-    // from the timer element.
-    private Object currentTimerKey;
 
     public SdkHarnessDoFnRunner(
         String mainInput,
@@ -548,17 +534,17 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
         BufferedOutputManager<OutputT> outputManager,
         Map<String, TupleTag<?>> outputMap,
         Coder<BoundedWindow> windowCoder,
-        KeySelector<WindowedValue<InputT>, ?> keySelector,
-        BiConsumer<WindowedValue<InputT>, TimerInternals.TimerData> timerRegistration) {
+        BiConsumer<WindowedValue<InputT>, TimerInternals.TimerData> timerRegistration,
+        Supplier<Object> keyForTimer) {
       this.mainInput = mainInput;
       this.stageBundleFactory = stageBundleFactory;
       this.stateRequestHandler = stateRequestHandler;
       this.progressHandler = progressHandler;
       this.outputManager = outputManager;
       this.outputMap = outputMap;
-      this.keySelector = keySelector;
       this.timerRegistration = timerRegistration;
       this.timerOutputIdToSpecMap = new HashMap<>();
+      this.keyForTimer = keyForTimer;
       // Gather all timers from all transforms by their output pCollectionId which is unique
       for (Map<String, ProcessBundleDescriptors.TimerSpec> transformTimerMap :
           stageBundleFactory.getProcessBundleDescriptor().getTimerSpecs().values()) {
@@ -609,8 +595,8 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
     @Override
     public void onTimer(
         String timerId, BoundedWindow window, Instant timestamp, TimeDomain timeDomain) {
-      Preconditions.checkNotNull(
-          currentTimerKey, "Key for timer needs to be set before calling onTimer");
+      Object timerKey = keyForTimer.get();
+      Preconditions.checkNotNull(timerKey, "Key for timer needs to be set before calling onTimer");
       Preconditions.checkNotNull(remoteBundle, "Call to onTimer outside of a bundle");
       LOG.debug("timer callback: {} {} {} {}", timerId, window, timestamp, timeDomain);
       FnDataReceiver<WindowedValue<?>> timerReceiver =
@@ -620,7 +606,7 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
               timerId);
       WindowedValue<KV<Object, Timer>> timerValue =
           WindowedValue.of(
-              KV.of(currentTimerKey, Timer.of(timestamp, new byte[0])),
+              KV.of(timerKey, Timer.of(timestamp, new byte[0])),
               timestamp,
               Collections.singleton(window),
               PaneInfo.NO_FIRING);
@@ -629,8 +615,6 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
       } catch (Exception e) {
         throw new RuntimeException(
             String.format(Locale.ENGLISH, "Failed to process timer %s", timerReceiver), e);
-      } finally {
-        currentTimerKey = null;
       }
     }
 
@@ -648,16 +632,6 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
       }
     }
 
-    /** Key for timer which has not been registered yet. */
-    Object getCurrentTimerKey() {
-      return currentTimerKey;
-    }
-
-    /** Key for timer which is about to be fired. */
-    void setCurrentTimerKey(Object key) {
-      this.currentTimerKey = key;
-    }
-
     boolean isBundleInProgress() {
       return remoteBundle != null;
     }
@@ -723,7 +697,6 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
             windowingStrategy,
             keyCoder,
             windowCoder,
-            sdkHarnessRunner::setCurrentTimerKey,
             getKeyedStateBackend());
 
     List<String> userStates =
@@ -744,7 +717,6 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
     private final WindowingStrategy windowingStrategy;
     private final Coder keyCoder;
     private final Coder windowCoder;
-    private final Consumer<ByteBuffer> currentKeyConsumer;
     private final KeyedStateBackend<ByteBuffer> keyedStateBackend;
 
     CleanupTimer(
@@ -753,14 +725,12 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
         WindowingStrategy windowingStrategy,
         Coder keyCoder,
         Coder windowCoder,
-        Consumer<ByteBuffer> currentKeyConsumer,
         KeyedStateBackend<ByteBuffer> keyedStateBackend) {
       this.timerInternals = timerInternals;
       this.stateBackendLock = stateBackendLock;
       this.windowingStrategy = windowingStrategy;
       this.keyCoder = keyCoder;
       this.windowCoder = windowCoder;
-      this.currentKeyConsumer = currentKeyConsumer;
       this.keyedStateBackend = keyedStateBackend;
     }
 
@@ -774,19 +744,10 @@ public class ExecutableStageDoFnOperator<InputT, OutputT> extends DoFnOperator<I
       Preconditions.checkNotNull(input, "Null input passed to CleanupTimer");
       // make sure this fires after any window.maxTimestamp() timers
       Instant gcTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy).plus(1);
-      final ByteBuffer key;
-      try {
-        key = ByteBuffer.wrap(CoderUtils.encodeToByteArray(keyCoder, ((KV) input).getKey()));
-      } catch (CoderException e) {
-        throw new RuntimeException("Failed to encode key for Flink state backend", e);
-      }
+      final ByteBuffer key = FlinkKeyUtils.encodeKey(((KV) input).getKey(), keyCoder);
       // Ensure the state backend is not concurrently accessed by the state requests
       try {
         stateBackendLock.lock();
-        // Set these two to ensure correct timer registration
-        // 1) For the timer setting
-        currentKeyConsumer.accept(key);
-        // 2) For the timer deduplication
         keyedStateBackend.setCurrentKey(key);
         timerInternals.setTimer(
             StateNamespaces.window(windowCoder, window),
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtils.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtils.java
new file mode 100644
index 0000000..687a231
--- /dev/null
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtils.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink.translation.wrappers.streaming;
+
+import static org.apache.beam.vendor.guava.v20_0.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.beam.vendor.guava.v20_0.com.google.common.base.Preconditions.checkState;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Locale;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.util.CoderUtils;
+
+/**
+ * Utility functions for dealing with key encoding. Beam requires keys to be compared in binary
+ * format. The helpers here ensure that a consistent encoding is used.
+ */
+class FlinkKeyUtils {
+
+  /** Encodes a key to a byte array wrapped inside a ByteBuffer. */
+  static <K> ByteBuffer encodeKey(K key, Coder<K> keyCoder) {
+    checkNotNull(keyCoder, "Provided coder must not be null");
+    final byte[] keyBytes;
+    try {
+      keyBytes = CoderUtils.encodeToByteArray(keyCoder, key);
+    } catch (Exception e) {
+      throw new RuntimeException(String.format(Locale.ENGLISH, "Failed to encode key: %s", key), e);
+    }
+    return ByteBuffer.wrap(keyBytes);
+  }
+
+  /** Decodes a key from a ByteBuffer containing a byte array. */
+  static <K> K decodeKey(ByteBuffer byteBuffer, Coder<K> keyCoder) {
+    checkNotNull(byteBuffer, "Provided ByteBuffer must not be null");
+    checkNotNull(keyCoder, "Provided coder must not be null");
+    checkState(byteBuffer.hasArray(), "ByteBuffer key must contain an array.");
+    @SuppressWarnings("ByteBufferBackingArray")
+    final byte[] keyBytes = byteBuffer.array();
+    try {
+      return CoderUtils.decodeFromByteArray(keyCoder, keyBytes);
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format(
+              Locale.ENGLISH, "Failed to decode encoded key: %s", Arrays.toString(keyBytes)),
+          e);
+    }
+  }
+}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/KvToByteBufferKeySelector.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/KvToByteBufferKeySelector.java
index 9438291..ea181d2 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/KvToByteBufferKeySelector.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/KvToByteBufferKeySelector.java
@@ -19,7 +19,6 @@ package org.apache.beam.runners.flink.translation.wrappers.streaming;
 
 import java.nio.ByteBuffer;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.util.CoderUtils;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -44,8 +43,7 @@ public class KvToByteBufferKeySelector<K, V>
   @Override
   public ByteBuffer getKey(WindowedValue<KV<K, V>> value) throws Exception {
     K key = value.getValue().getKey();
-    byte[] keyBytes = CoderUtils.encodeToByteArray(keyCoder, key);
-    return ByteBuffer.wrap(keyBytes);
+    return FlinkKeyUtils.encodeKey(key, keyCoder);
   }
 
   @Override
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WorkItemKeySelector.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WorkItemKeySelector.java
index ccc1ed4..ec77481 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WorkItemKeySelector.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WorkItemKeySelector.java
@@ -20,7 +20,6 @@ package org.apache.beam.runners.flink.translation.wrappers.streaming;
 import java.nio.ByteBuffer;
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.util.CoderUtils;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
@@ -45,8 +44,7 @@ public class WorkItemKeySelector<K, V>
   @Override
   public ByteBuffer getKey(WindowedValue<SingletonKeyedWorkItem<K, V>> value) throws Exception {
     K key = value.getValue().key();
-    byte[] keyBytes = CoderUtils.encodeToByteArray(keyCoder, key);
-    return ByteBuffer.wrap(keyBytes);
+    return FlinkKeyUtils.encodeKey(key, keyCoder);
   }
 
   @Override
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperatorTest.java
index 2b51583..1e5d120 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperatorTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperatorTest.java
@@ -36,7 +36,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.locks.Lock;
-import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
 import org.apache.beam.model.pipeline.v1.RunnerApi.ExecutableStagePayload;
@@ -66,7 +65,6 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.state.BagState;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.CoderUtils;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
@@ -395,7 +393,6 @@ public class ExecutableStageDoFnOperatorTest {
   @Test
   public void testEnsureStateCleanupWithKeyedInputCleanupTimer() throws Exception {
     InMemoryTimerInternals inMemoryTimerInternals = new InMemoryTimerInternals();
-    Consumer<ByteBuffer> keyConsumer = Mockito.mock(Consumer.class);
     KeyedStateBackend keyedStateBackend = Mockito.mock(KeyedStateBackend.class);
     Lock stateBackendLock = Mockito.mock(Lock.class);
     StringUtf8Coder keyCoder = StringUtf8Coder.of();
@@ -410,13 +407,11 @@ public class ExecutableStageDoFnOperatorTest {
             WindowingStrategy.globalDefault(),
             keyCoder,
             windowCoder,
-            keyConsumer,
             keyedStateBackend);
     cleanupTimer.setForWindow(KV.of("key", "string"), window);
 
     Mockito.verify(stateBackendLock).lock();
-    ByteBuffer key = ByteBuffer.wrap(CoderUtils.encodeToByteArray(keyCoder, "key"));
-    Mockito.verify(keyConsumer).accept(key);
+    ByteBuffer key = FlinkKeyUtils.encodeKey("key", keyCoder);
     Mockito.verify(keyedStateBackend).setCurrentKey(key);
     assertThat(
         inMemoryTimerInternals.getNextTimer(TimeDomain.EVENT_TIME),
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtilsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtilsTest.java
new file mode 100644
index 0000000..06b5d01
--- /dev/null
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtilsTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink.translation.wrappers.streaming;
+
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+import com.google.protobuf.ByteString;
+import java.nio.ByteBuffer;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.extensions.protobuf.ByteStringCoder;
+import org.junit.Test;
+
+/** Tests for {@link FlinkKeyUtils}. */
+public class FlinkKeyUtilsTest {
+
+  @Test
+  public void testEncodeDecode() {
+    String key = "key";
+    StringUtf8Coder coder = StringUtf8Coder.of();
+
+    ByteBuffer byteBuffer = FlinkKeyUtils.encodeKey(key, coder);
+    assertThat(FlinkKeyUtils.decodeKey(byteBuffer, coder), is(key));
+  }
+
+  @Test
+  public void testNullKey() {
+    Void key = null;
+    VoidCoder coder = VoidCoder.of();
+
+    ByteBuffer byteBuffer = FlinkKeyUtils.encodeKey(key, coder);
+    assertThat(FlinkKeyUtils.decodeKey(byteBuffer, coder), is(nullValue()));
+  }
+
+  @Test
+  @SuppressWarnings("ByteBufferBackingArray")
+  public void testCoderContext() throws Exception {
+    byte[] bytes = {1, 1, 1};
+    ByteString key = ByteString.copyFrom(bytes);
+    ByteStringCoder coder = ByteStringCoder.of();
+
+    ByteBuffer encoded = FlinkKeyUtils.encodeKey(key, coder);
+    // Ensure outer context is used where no length encoding is used.
+    assertThat(encoded.array(), is(bytes));
+  }
+}