You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2020/02/10 03:43:29 UTC
[flink-statefun] branch master updated (0b60796 -> 1158cd1)
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git.
from 0b60796 [FLINK-15952] Replace byte[] with MultiplexedStateKey
new 840b005 [FLINK-15954] Add PersistedTable SDK classes
new 45f4d17 [FLINK-15954] Add PersistedTable to bound state
new 1ebe254 [FLINK-15954] Add a simple LRU cache
new 348628b [FLINK-15954] Support PersistedTable in StateBinder
new ea60401 [FLINK-15954] Add Flink based TableAccessors
new 65ed306 [FLINK-15954] Add PersistedTable support in statefun runtime
new ae9b5db [FLINK-15955] Add an HttpFunctionSpec
new 5686463 [FLINK-15955] Rename RemoteFunction* -> GrpcFunction*
new 1158cd1 [FLINK-15955] Configure a remote function acorrding to its spec
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../flink/core/cache/SingleThreadedLruCache.java | 63 +++++++++
.../{ModuleType.java => FunctionSpec.java} | 14 +-
.../{RemoteFunction.java => GrpcFunction.java} | 6 +-
...tionProvider.java => GrpcFunctionProvider.java} | 10 +-
...moteFunctionSpec.java => GrpcFunctionSpec.java} | 14 +-
.../core/jsonmodule/HttpFunctionProvider.java | 13 +-
...moteFunctionSpec.java => HttpFunctionSpec.java} | 29 ++--
.../statefun/flink/core/jsonmodule/JsonModule.java | 78 ++++++++---
.../statefun/flink/core/jsonmodule/Pointers.java | 4 +
.../statefun/flink/core/state/BoundState.java | 44 +++++-
.../statefun/flink/core/state/FlinkState.java | 16 +++
...kValueAccessor.java => FlinkTableAccessor.java} | 35 +++--
.../core/state/MultiplexedMapStateAccessor.java | 28 ----
.../flink/core/state/MultiplexedState.java | 15 +++
.../core/state/MultiplexedTableStateAccessor.java | 112 ++++++++++++++++
.../{PersistedValues.java => PersistedStates.java} | 30 ++---
...{FlinkValueAccessor.java => RawSerializer.java} | 51 +++----
.../flink/statefun/flink/core/state/State.java | 5 +
.../statefun/flink/core/state/StateBinder.java | 36 ++++-
.../flink/statefun/sdk/state/ApiExtension.java | 5 +
.../core/cache/SingleThreadedLruCacheTest.java} | 36 ++---
.../statefun/flink/core/state/StateBinderTest.java | 41 ++++++
.../src/test/resources/bar-module/module.yaml | 5 +-
.../flink/statefun/sdk/state/PersistedTable.java | 148 +++++++++++++++++++++
.../state/{Accessor.java => TableAccessor.java} | 8 +-
25 files changed, 679 insertions(+), 167 deletions(-)
create mode 100644 statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/cache/SingleThreadedLruCache.java
copy statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/{ModuleType.java => FunctionSpec.java} (83%)
rename statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/{RemoteFunction.java => GrpcFunction.java} (88%)
rename statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/{RemoteFunctionProvider.java => GrpcFunctionProvider.java} (79%)
copy statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/{RemoteFunctionSpec.java => GrpcFunctionSpec.java} (81%)
copy statefun-docs/src/main/java/org/apache/flink/statefun/docs/CustomProvider.java => statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/HttpFunctionProvider.java (74%)
rename statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/{RemoteFunctionSpec.java => HttpFunctionSpec.java} (65%)
copy statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/{FlinkValueAccessor.java => FlinkTableAccessor.java} (65%)
create mode 100644 statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedTableStateAccessor.java
rename statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/{PersistedValues.java => PersistedStates.java} (75%)
copy statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/{FlinkValueAccessor.java => RawSerializer.java} (50%)
copy statefun-flink/{statefun-flink-io-bundle/src/test/java/org/apache/flink/statefun/flink/io/common/ReflectionUtilTest.java => statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/cache/SingleThreadedLruCacheTest.java} (50%)
create mode 100644 statefun-sdk/src/main/java/org/apache/flink/statefun/sdk/state/PersistedTable.java
copy statefun-sdk/src/main/java/org/apache/flink/statefun/sdk/state/{Accessor.java => TableAccessor.java} (89%)
[flink-statefun] 03/09: [FLINK-15954] Add a simple LRU cache
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit 1ebe254fe53b26d1f64a7c06637161927b2e1dc9
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Fri Feb 7 17:35:24 2020 +0100
[FLINK-15954] Add a simple LRU cache
---
.../flink/core/cache/SingleThreadedLruCache.java | 63 ++++++++++++++++++++++
.../core/cache/SingleThreadedLruCacheTest.java | 51 ++++++++++++++++++
2 files changed, 114 insertions(+)
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/cache/SingleThreadedLruCache.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/cache/SingleThreadedLruCache.java
new file mode 100644
index 0000000..2084eeb
--- /dev/null
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/cache/SingleThreadedLruCache.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.statefun.flink.core.cache;
+
+import java.util.LinkedHashMap;
+import java.util.Map.Entry;
+import java.util.function.Function;
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.NotThreadSafe;
+
+@NotThreadSafe
+public final class SingleThreadedLruCache<K, V> {
+ private final Cache<K, V> cache;
+
+ public SingleThreadedLruCache(int maxCapacity) {
+ this.cache = new Cache<>(maxCapacity, maxCapacity);
+ }
+
+ public void put(K key, V value) {
+ cache.put(key, value);
+ }
+
+ @Nullable
+ public V get(K key) {
+ return cache.get(key);
+ }
+
+ public V computeIfAbsent(K key, Function<? super K, ? extends V> mappingFunction) {
+ return cache.computeIfAbsent(key, mappingFunction);
+ }
+
+ private static final class Cache<K, V> extends LinkedHashMap<K, V> {
+
+ private static final long serialVersionUID = 1;
+
+ private final int maxCapacity;
+
+ private Cache(int initialCapacity, int maxCapacity) {
+ super(initialCapacity, 0.75f, true);
+ this.maxCapacity = maxCapacity;
+ }
+
+ @Override
+ protected boolean removeEldestEntry(Entry<K, V> eldest) {
+ return size() > maxCapacity;
+ }
+ }
+}
diff --git a/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/cache/SingleThreadedLruCacheTest.java b/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/cache/SingleThreadedLruCacheTest.java
new file mode 100644
index 0000000..5059f94
--- /dev/null
+++ b/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/cache/SingleThreadedLruCacheTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.statefun.flink.core.cache;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+import org.junit.Test;
+
+public class SingleThreadedLruCacheTest {
+
+ @Test
+ public void exampleUsage() {
+ SingleThreadedLruCache<String, String> cache = new SingleThreadedLruCache<>(2);
+
+ cache.put("a", "1");
+ cache.put("b", "2");
+
+ assertThat(cache.get("a"), is("1"));
+ assertThat(cache.get("b"), is("2"));
+ }
+
+ @Test
+ public void leastRecentlyElementShouldBeEvicted() {
+ SingleThreadedLruCache<String, String> cache = new SingleThreadedLruCache<>(2);
+
+ cache.put("a", "1");
+ cache.put("b", "2");
+ cache.put("c", "3");
+
+ assertThat(cache.get("a"), nullValue());
+ assertThat(cache.get("b"), is("2"));
+ assertThat(cache.get("c"), is("3"));
+ }
+}
[flink-statefun] 04/09: [FLINK-15954] Support PersistedTable in
StateBinder
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit 348628be4fe9d35ea1c3a9802b53029a7370be73
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Fri Feb 7 17:39:18 2020 +0100
[FLINK-15954] Support PersistedTable in StateBinder
---
.../{PersistedValues.java => PersistedStates.java} | 30 +++++++---------
.../statefun/flink/core/state/StateBinder.java | 36 +++++++++++++++----
.../flink/statefun/sdk/state/ApiExtension.java | 5 +++
.../statefun/flink/core/state/StateBinderTest.java | 41 ++++++++++++++++++++++
4 files changed, 89 insertions(+), 23 deletions(-)
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/PersistedValues.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/PersistedStates.java
similarity index 75%
rename from statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/PersistedValues.java
rename to statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/PersistedStates.java
index 3c12c8b..8b401bf 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/PersistedValues.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/PersistedStates.java
@@ -27,17 +27,18 @@ import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.flink.statefun.sdk.annotations.Persisted;
+import org.apache.flink.statefun.sdk.state.PersistedTable;
import org.apache.flink.statefun.sdk.state.PersistedValue;
-final class PersistedValues {
+final class PersistedStates {
- static List<PersistedValue<Object>> findReflectively(@Nullable Object instance) {
- PersistedValues visitor = new PersistedValues();
+ static List<?> findReflectively(@Nullable Object instance) {
+ PersistedStates visitor = new PersistedStates();
visitor.visit(instance);
return visitor.getPersistedValues();
}
- private final List<PersistedValue<Object>> persistedValues = new ArrayList<>();
+ private final List<Object> persistedValues = new ArrayList<>();
private void visit(@Nullable Object instance) {
if (instance == null) {
@@ -48,17 +49,14 @@ final class PersistedValues {
}
}
- private List<PersistedValue<Object>> getPersistedValues() {
+ private List<Object> getPersistedValues() {
return persistedValues;
}
private void visitField(@Nonnull Object instance, @Nonnull Field field) {
- if (field.getType() != PersistedValue.class) {
+ if (field.getType() != PersistedValue.class && field.getType() != PersistedTable.class) {
throw new IllegalArgumentException(
- "Unknown persisted value type "
- + field.getType()
- + " on "
- + instance.getClass().getName());
+ "Unknown persisted type " + field.getType() + " on " + instance.getClass().getName());
}
if (Modifier.isStatic(field.getModifiers())) {
throw new IllegalArgumentException(
@@ -67,7 +65,7 @@ final class PersistedValues {
+ " on "
+ instance.getClass().getName());
}
- PersistedValue<Object> persistedValue = getPersistedValueReflectively(instance, field);
+ Object persistedValue = getPersistedValueReflectively(instance, field);
if (persistedValue == null) {
throw new IllegalStateException(
"The field " + field + " of a " + instance.getClass().getName() + " was not initialized");
@@ -75,15 +73,13 @@ final class PersistedValues {
persistedValues.add(persistedValue);
}
- @SuppressWarnings("unchecked")
- private static PersistedValue<Object> getPersistedValueReflectively(
- Object instance, Field persistedValueField) {
+ private static Object getPersistedValueReflectively(Object instance, Field persistedField) {
try {
- persistedValueField.setAccessible(true);
- return (PersistedValue<Object>) persistedValueField.get(instance);
+ persistedField.setAccessible(true);
+ return persistedField.get(instance);
} catch (IllegalAccessException e) {
throw new RuntimeException(
- "Unable access field " + persistedValueField.getName() + " of " + instance.getClass());
+ "Unable access field " + persistedField.getName() + " of " + instance.getClass());
}
}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/StateBinder.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/StateBinder.java
index 433cc8e..a7b0c4c 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/StateBinder.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/StateBinder.java
@@ -22,10 +22,13 @@ import java.util.Objects;
import javax.annotation.Nullable;
import org.apache.flink.statefun.flink.core.di.Inject;
import org.apache.flink.statefun.flink.core.di.Label;
+import org.apache.flink.statefun.flink.core.state.BoundState.Builder;
import org.apache.flink.statefun.sdk.FunctionType;
import org.apache.flink.statefun.sdk.state.Accessor;
import org.apache.flink.statefun.sdk.state.ApiExtension;
+import org.apache.flink.statefun.sdk.state.PersistedTable;
import org.apache.flink.statefun.sdk.state.PersistedValue;
+import org.apache.flink.statefun.sdk.state.TableAccessor;
public final class StateBinder {
private final State state;
@@ -36,13 +39,34 @@ public final class StateBinder {
}
public BoundState bind(FunctionType functionType, @Nullable Object instance) {
- List<PersistedValue<Object>> values = PersistedValues.findReflectively(instance);
-
- for (PersistedValue<Object> persistedValue : values) {
- Accessor<Object> accessor = state.createFlinkStateAccessor(functionType, persistedValue);
- ApiExtension.setPersistedValueAccessor(persistedValue, accessor);
+ List<?> states = PersistedStates.findReflectively(instance);
+ Builder builder = BoundState.builder();
+ for (Object persisted : states) {
+ if (persisted instanceof PersistedValue) {
+ PersistedValue<?> persistedValue = (PersistedValue<?>) persisted;
+ Accessor<?> accessor = state.createFlinkStateAccessor(functionType, persistedValue);
+ setAccessorRaw(persistedValue, accessor);
+ builder.withPersistedValue(persistedValue);
+ } else if (persisted instanceof PersistedTable) {
+ PersistedTable<?, ?> persistedTable = (PersistedTable<?, ?>) persisted;
+ TableAccessor<?, ?> accessor =
+ state.createFlinkStateTableAccessor(functionType, persistedTable);
+ setAccessorRaw(persistedTable, accessor);
+ builder.withPersistedTable(persistedTable);
+ } else {
+ throw new IllegalArgumentException("Unknown persisted field " + persisted);
+ }
}
+ return builder.build();
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private void setAccessorRaw(PersistedTable<?, ?> persistedTable, TableAccessor<?, ?> accessor) {
+ ApiExtension.setPersistedTableAccessor((PersistedTable) persistedTable, accessor);
+ }
- return new BoundState(values);
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private static void setAccessorRaw(PersistedValue<?> persistedValue, Accessor<?> accessor) {
+ ApiExtension.setPersistedValueAccessor((PersistedValue) persistedValue, accessor);
}
}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/sdk/state/ApiExtension.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/sdk/state/ApiExtension.java
index 3f079b9..d8cefd4 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/sdk/state/ApiExtension.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/sdk/state/ApiExtension.java
@@ -22,4 +22,9 @@ public class ApiExtension {
PersistedValue<T> persistedValue, Accessor<T> accessor) {
persistedValue.setAccessor(accessor);
}
+
+ public static <K, V> void setPersistedTableAccessor(
+ PersistedTable<K, V> persistedTable, TableAccessor<K, V> accessor) {
+ persistedTable.setAccessor(accessor);
+ }
}
diff --git a/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/state/StateBinderTest.java b/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/state/StateBinderTest.java
index c040138..32cf8d5 100644
--- a/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/state/StateBinderTest.java
+++ b/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/state/StateBinderTest.java
@@ -21,14 +21,18 @@ import static org.hamcrest.CoreMatchers.hasItems;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertTrue;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
import org.apache.flink.statefun.flink.core.TestUtils;
import org.apache.flink.statefun.sdk.Address;
import org.apache.flink.statefun.sdk.FunctionType;
import org.apache.flink.statefun.sdk.annotations.Persisted;
import org.apache.flink.statefun.sdk.state.Accessor;
+import org.apache.flink.statefun.sdk.state.PersistedTable;
import org.apache.flink.statefun.sdk.state.PersistedValue;
+import org.apache.flink.statefun.sdk.state.TableAccessor;
import org.junit.Test;
public class StateBinderTest {
@@ -75,6 +79,13 @@ public class StateBinderTest {
binderUnderTest.bind(TestUtils.FUNCTION_TYPE, new StaticPersistedValue());
}
+ @Test
+ public void bindPersistedTable() {
+ binderUnderTest.bind(TestUtils.FUNCTION_TYPE, new PersistedTableValue());
+
+ assertThat(state.boundNames, hasItems("table"));
+ }
+
static final class SanityClass {
@SuppressWarnings("unused")
@@ -124,6 +135,12 @@ public class StateBinderTest {
static PersistedValue<String> value = PersistedValue.of("static", String.class);
}
+ static final class PersistedTableValue {
+ @Persisted
+ @SuppressWarnings("unused")
+ PersistedTable<String, byte[]> value = PersistedTable.of("table", String.class, byte[].class);
+ }
+
private static final class FakeState implements State {
Set<String> boundNames = new HashSet<>();
@@ -153,6 +170,30 @@ public class StateBinderTest {
}
@Override
+ public <K, V> TableAccessor<K, V> createFlinkStateTableAccessor(
+ FunctionType functionType, PersistedTable<K, V> persistedTable) {
+ boundNames.add(persistedTable.name());
+ return new TableAccessor<K, V>() {
+ Map<K, V> map = new HashMap<>();
+
+ @Override
+ public void set(K key, V value) {
+ map.put(key, value);
+ }
+
+ @Override
+ public V get(K key) {
+ return map.get(key);
+ }
+
+ @Override
+ public void remove(K key) {
+ map.remove(key);
+ }
+ };
+ }
+
+ @Override
public void setCurrentKey(Address key) {
throw new UnsupportedOperationException();
}
[flink-statefun] 08/09: [FLINK-15955] Rename RemoteFunction* ->
GrpcFunction*
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit 5686463495365acad3af336307f79ee92d47a853
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Fri Feb 7 18:22:32 2020 +0100
[FLINK-15955] Rename RemoteFunction* -> GrpcFunction*
---
.../jsonmodule/{RemoteFunction.java => GrpcFunction.java} | 6 +++---
...moteFunctionProvider.java => GrpcFunctionProvider.java} | 10 +++++-----
.../{RemoteFunctionSpec.java => GrpcFunctionSpec.java} | 14 ++++++++++----
.../flink/statefun/flink/core/jsonmodule/JsonModule.java | 10 +++++-----
.../src/test/resources/bar-module/module.yaml | 5 +++--
5 files changed, 26 insertions(+), 19 deletions(-)
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/RemoteFunction.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/GrpcFunction.java
similarity index 88%
rename from statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/RemoteFunction.java
rename to statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/GrpcFunction.java
index 8d24dd5..c7e7d51 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/RemoteFunction.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/GrpcFunction.java
@@ -21,10 +21,10 @@ import java.util.Objects;
import org.apache.flink.statefun.sdk.Context;
import org.apache.flink.statefun.sdk.StatefulFunction;
-public class RemoteFunction implements StatefulFunction {
- private final RemoteFunctionSpec functionSpec;
+public class GrpcFunction implements StatefulFunction {
+ private final GrpcFunctionSpec functionSpec;
- public RemoteFunction(RemoteFunctionSpec functionSpec) {
+ public GrpcFunction(GrpcFunctionSpec functionSpec) {
this.functionSpec = Objects.requireNonNull(functionSpec);
}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/RemoteFunctionProvider.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/GrpcFunctionProvider.java
similarity index 79%
rename from statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/RemoteFunctionProvider.java
rename to statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/GrpcFunctionProvider.java
index f088161..4d5db34 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/RemoteFunctionProvider.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/GrpcFunctionProvider.java
@@ -22,19 +22,19 @@ import org.apache.flink.statefun.sdk.FunctionType;
import org.apache.flink.statefun.sdk.StatefulFunction;
import org.apache.flink.statefun.sdk.StatefulFunctionProvider;
-public class RemoteFunctionProvider implements StatefulFunctionProvider {
- private final Map<FunctionType, RemoteFunctionSpec> supportedTypes;
+public class GrpcFunctionProvider implements StatefulFunctionProvider {
+ private final Map<FunctionType, GrpcFunctionSpec> supportedTypes;
- public RemoteFunctionProvider(Map<FunctionType, RemoteFunctionSpec> supportedTypes) {
+ public GrpcFunctionProvider(Map<FunctionType, GrpcFunctionSpec> supportedTypes) {
this.supportedTypes = supportedTypes;
}
@Override
public StatefulFunction functionOfType(FunctionType type) {
- RemoteFunctionSpec spec = supportedTypes.get(type);
+ GrpcFunctionSpec spec = supportedTypes.get(type);
if (spec == null) {
throw new IllegalArgumentException("Unsupported type " + type);
}
- return new RemoteFunction(spec);
+ return new GrpcFunction(spec);
}
}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/RemoteFunctionSpec.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/GrpcFunctionSpec.java
similarity index 81%
rename from statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/RemoteFunctionSpec.java
rename to statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/GrpcFunctionSpec.java
index 4cc0025..62bb4f8 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/RemoteFunctionSpec.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/GrpcFunctionSpec.java
@@ -21,20 +21,26 @@ import java.net.SocketAddress;
import java.util.Objects;
import org.apache.flink.statefun.sdk.FunctionType;
-final class RemoteFunctionSpec {
+final class GrpcFunctionSpec implements FunctionSpec {
private final FunctionType functionType;
private final SocketAddress functionAddress;
- RemoteFunctionSpec(FunctionType functionType, SocketAddress functionAddress) {
+ GrpcFunctionSpec(FunctionType functionType, SocketAddress functionAddress) {
this.functionType = Objects.requireNonNull(functionType);
this.functionAddress = Objects.requireNonNull(functionAddress);
}
- FunctionType functionType() {
+ @Override
+ public FunctionType functionType() {
return functionType;
}
- SocketAddress address() {
+ @Override
+ public Kind kind() {
+ return Kind.GRPC;
+ }
+
+ public SocketAddress address() {
return functionAddress;
}
}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/JsonModule.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/JsonModule.java
index 3a15f76..4465864 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/JsonModule.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/JsonModule.java
@@ -64,15 +64,15 @@ final class JsonModule implements StatefulFunctionModule {
}
private void configureFunctions(Binder binder, Iterable<? extends JsonNode> functions) {
- final Map<FunctionType, RemoteFunctionSpec> definedFunctions =
+ final Map<FunctionType, GrpcFunctionSpec> definedFunctions =
StreamSupport.stream(functions.spliterator(), false)
.map(JsonModule::parseRemoteFunctionSpec)
- .collect(toMap(RemoteFunctionSpec::functionType, Function.identity()));
+ .collect(toMap(GrpcFunctionSpec::functionType, Function.identity()));
// currently we had a single function type that can be specified at a module.yaml
// and this is the RemoteFunction. Therefore we translate immediately the function spec
// to a (GRPC) RemoteFunctionProvider.
- RemoteFunctionProvider provider = new RemoteFunctionProvider(definedFunctions);
+ GrpcFunctionProvider provider = new GrpcFunctionProvider(definedFunctions);
for (FunctionType type : definedFunctions.keySet()) {
binder.bindFunctionProvider(type, provider);
}
@@ -168,10 +168,10 @@ final class JsonModule implements StatefulFunctionModule {
// Functions
// ----------------------------------------------------------------------------------------------------------
- private static RemoteFunctionSpec parseRemoteFunctionSpec(JsonNode functionNode) {
+ private static GrpcFunctionSpec parseRemoteFunctionSpec(JsonNode functionNode) {
FunctionType functionType = functionType(functionNode);
InetSocketAddress functionAddress = functionAddress(functionNode);
- return new RemoteFunctionSpec(functionType, functionAddress);
+ return new GrpcFunctionSpec(functionType, functionAddress);
}
private static FunctionType functionType(JsonNode functionNode) {
diff --git a/statefun-flink/statefun-flink-core/src/test/resources/bar-module/module.yaml b/statefun-flink/statefun-flink-core/src/test/resources/bar-module/module.yaml
index fe345f2..c659fe9 100644
--- a/statefun-flink/statefun-flink-core/src/test/resources/bar-module/module.yaml
+++ b/statefun-flink/statefun-flink-core/src/test/resources/bar-module/module.yaml
@@ -20,16 +20,17 @@ module:
functions:
- function:
meta:
+ kind: grpc
type: com.example/hello
spec:
host: localhost
port: 5000
- function:
meta:
+ kind: http
type: com.foo/world
spec:
- host: localhost
- port: 8855
+ endpoint: localhost:5959/statefun
routers:
- router:
meta:
[flink-statefun] 09/09: [FLINK-15955] Configure a remote function
acorrding to its spec
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit 1158cd18b3a357de7b3265115e05bd278619acb3
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Fri Feb 7 18:35:28 2020 +0100
[FLINK-15955] Configure a remote function acorrding to its spec
This commit would instantiate the correct remote function
provider with its corresponding function spec, as specified
in the module.yaml.
This closes #18.
---
.../core/jsonmodule/HttpFunctionProvider.java | 33 +++++++++
.../statefun/flink/core/jsonmodule/JsonModule.java | 78 +++++++++++++++++-----
2 files changed, 96 insertions(+), 15 deletions(-)
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/HttpFunctionProvider.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/HttpFunctionProvider.java
new file mode 100644
index 0000000..58c1c7c
--- /dev/null
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/HttpFunctionProvider.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.statefun.flink.core.jsonmodule;
+
+import java.util.Map;
+import org.apache.flink.statefun.sdk.FunctionType;
+import org.apache.flink.statefun.sdk.StatefulFunction;
+import org.apache.flink.statefun.sdk.StatefulFunctionProvider;
+
+public class HttpFunctionProvider implements StatefulFunctionProvider {
+ public HttpFunctionProvider(Map<FunctionType, HttpFunctionSpec> specs) {}
+
+ @Override
+ public StatefulFunction functionOfType(FunctionType type) {
+ throw new UnsupportedOperationException("Not implemented yet");
+ }
+}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/JsonModule.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/JsonModule.java
index 4465864..7f2219f 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/JsonModule.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/JsonModule.java
@@ -18,27 +18,37 @@
package org.apache.flink.statefun.flink.core.jsonmodule;
import static java.lang.String.format;
+import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toMap;
+import static org.apache.flink.statefun.flink.core.common.Maps.transformValues;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.net.URI;
import java.net.URL;
+import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import java.util.function.Function;
+import java.util.stream.Collector;
import java.util.stream.StreamSupport;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
import org.apache.flink.statefun.flink.common.ResourceLocator;
import org.apache.flink.statefun.flink.common.json.NamespaceNamePair;
import org.apache.flink.statefun.flink.common.json.Selectors;
import org.apache.flink.statefun.flink.common.protobuf.ProtobufDescriptorMap;
+import org.apache.flink.statefun.flink.core.jsonmodule.FunctionSpec.Kind;
import org.apache.flink.statefun.flink.core.protorouter.ProtobufRouter;
import org.apache.flink.statefun.flink.io.spi.JsonIngressSpec;
import org.apache.flink.statefun.sdk.FunctionType;
import org.apache.flink.statefun.sdk.IngressType;
+import org.apache.flink.statefun.sdk.StatefulFunctionProvider;
import org.apache.flink.statefun.sdk.io.IngressIdentifier;
import org.apache.flink.statefun.sdk.io.Router;
import org.apache.flink.statefun.sdk.spi.StatefulFunctionModule;
@@ -64,17 +74,31 @@ final class JsonModule implements StatefulFunctionModule {
}
private void configureFunctions(Binder binder, Iterable<? extends JsonNode> functions) {
- final Map<FunctionType, GrpcFunctionSpec> definedFunctions =
+ final Map<Kind, Map<FunctionType, FunctionSpec>> definedFunctions =
StreamSupport.stream(functions.spliterator(), false)
- .map(JsonModule::parseRemoteFunctionSpec)
- .collect(toMap(GrpcFunctionSpec::functionType, Function.identity()));
-
- // currently we had a single function type that can be specified at a module.yaml
- // and this is the RemoteFunction. Therefore we translate immediately the function spec
- // to a (GRPC) RemoteFunctionProvider.
- GrpcFunctionProvider provider = new GrpcFunctionProvider(definedFunctions);
- for (FunctionType type : definedFunctions.keySet()) {
- binder.bindFunctionProvider(type, provider);
+ .map(JsonModule::parseFunctionSpec)
+ .collect(groupingBy(FunctionSpec::kind, groupByFunctionType()));
+
+ for (Entry<Kind, Map<FunctionType, FunctionSpec>> entry : definedFunctions.entrySet()) {
+ StatefulFunctionProvider provider = functionProvider(entry.getKey(), entry.getValue());
+ Set<FunctionType> functionTypes = entry.getValue().keySet();
+ for (FunctionType type : functionTypes) {
+ binder.bindFunctionProvider(type, provider);
+ }
+ }
+ }
+
+ private static StatefulFunctionProvider functionProvider(
+ Kind kind, Map<FunctionType, FunctionSpec> definedFunctions) {
+ switch (kind) {
+ case HTTP:
+ return new HttpFunctionProvider(
+ transformValues(definedFunctions, HttpFunctionSpec.class::cast));
+ case GRPC:
+ return new GrpcFunctionProvider(
+ transformValues(definedFunctions, GrpcFunctionSpec.class::cast));
+ default:
+ throw new IllegalStateException("Unexpected value: " + kind);
}
}
@@ -103,7 +127,6 @@ final class JsonModule implements StatefulFunctionModule {
// ----------------------------------------------------------------------------------------------------------
// Ingresses
// ----------------------------------------------------------------------------------------------------------
-
private static IngressType ingressType(JsonNode spec) {
String typeString = Selectors.textAt(spec, Pointers.Ingress.META_TYPE);
NamespaceNamePair nn = NamespaceNamePair.from(typeString);
@@ -119,7 +142,6 @@ final class JsonModule implements StatefulFunctionModule {
// ----------------------------------------------------------------------------------------------------------
// Routers
// ----------------------------------------------------------------------------------------------------------
-
private static Router<Message> dynamicRouter(JsonNode router) {
String addressTemplate = Selectors.textAt(router, Pointers.Routers.SPEC_TARGET);
String descriptorSetPath = Selectors.textAt(router, Pointers.Routers.SPEC_DESCRIPTOR);
@@ -142,6 +164,10 @@ final class JsonModule implements StatefulFunctionModule {
private static ProtobufDescriptorMap protobufDescriptorMap(String descriptorSetPath) {
try {
URL url = ResourceLocator.findNamedResource(descriptorSetPath);
+ if (url == null) {
+ throw new IllegalArgumentException(
+ "Unable to locate a Protobuf descriptor set at " + descriptorSetPath);
+ }
return ProtobufDescriptorMap.from(url);
} catch (IOException e) {
throw new IllegalStateException(
@@ -168,10 +194,23 @@ final class JsonModule implements StatefulFunctionModule {
// Functions
// ----------------------------------------------------------------------------------------------------------
- private static GrpcFunctionSpec parseRemoteFunctionSpec(JsonNode functionNode) {
+ private static FunctionSpec parseFunctionSpec(JsonNode functionNode) {
+ String functionKind = Selectors.textAt(functionNode, Pointers.Functions.META_KIND);
+ FunctionSpec.Kind kind = Kind.valueOf(functionKind.toUpperCase(Locale.getDefault()));
FunctionType functionType = functionType(functionNode);
- InetSocketAddress functionAddress = functionAddress(functionNode);
- return new GrpcFunctionSpec(functionType, functionAddress);
+ switch (kind) {
+ case HTTP:
+ return new HttpFunctionSpec(
+ functionType, functionUri(functionNode), functionStates(functionNode));
+ case GRPC:
+ return new GrpcFunctionSpec(functionType, functionAddress(functionNode));
+ default:
+ throw new IllegalArgumentException("Unrecognized function kind " + functionKind);
+ }
+ }
+
+ private static List<String> functionStates(JsonNode functionNode) {
+ return Selectors.textListAt(functionNode, Pointers.Functions.FUNCTION_STATES);
}
private static FunctionType functionType(JsonNode functionNode) {
@@ -185,4 +224,13 @@ final class JsonModule implements StatefulFunctionModule {
int port = Selectors.integerAt(functionNode, Pointers.Functions.FUNCTION_PORT);
return new InetSocketAddress(host, port);
}
+
+ private static URI functionUri(JsonNode functionNode) {
+ String uri = Selectors.textAt(functionNode, Pointers.Functions.FUNCTION_ENDPOINT);
+ return URI.create(uri);
+ }
+
+ private static Collector<FunctionSpec, ?, Map<FunctionType, FunctionSpec>> groupByFunctionType() {
+ return toMap(FunctionSpec::functionType, Function.identity());
+ }
}
[flink-statefun] 05/09: [FLINK-15954] Add Flink based TableAccessors
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit ea60401f939257045ee52d04d7b69f26db5489df
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Fri Feb 7 17:41:09 2020 +0100
[FLINK-15954] Add Flink based TableAccessors
This commit adds two implementations of a TableAccessor interface,
one for the multiplexed state, and one for the non multiplexed.
This commit also extracts the RawSerializer and reuses it
across the multiplexed state accessors.
---
.../flink/core/state/FlinkTableAccessor.java | 58 +++++++++++
.../core/state/MultiplexedMapStateAccessor.java | 28 ------
.../core/state/MultiplexedTableStateAccessor.java | 112 +++++++++++++++++++++
.../statefun/flink/core/state/RawSerializer.java | 50 +++++++++
4 files changed, 220 insertions(+), 28 deletions(-)
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/FlinkTableAccessor.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/FlinkTableAccessor.java
new file mode 100644
index 0000000..258135c
--- /dev/null
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/FlinkTableAccessor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.statefun.flink.core.state;
+
+import java.util.Objects;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.statefun.sdk.state.TableAccessor;
+
+final class FlinkTableAccessor<K, V> implements TableAccessor<K, V> {
+
+ private final MapState<K, V> handle;
+
+ FlinkTableAccessor(MapState<K, V> handle) {
+ this.handle = Objects.requireNonNull(handle);
+ }
+
+ @Override
+ public void set(K key, V value) {
+ try {
+ handle.put(key, value);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public V get(K key) {
+ try {
+ return handle.get(key);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void remove(K key) {
+ try {
+ handle.remove(key);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedMapStateAccessor.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedMapStateAccessor.java
index 95ffe5e..986ef9f 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedMapStateAccessor.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedMapStateAccessor.java
@@ -17,12 +17,9 @@
*/
package org.apache.flink.statefun.flink.core.state;
-import java.io.IOException;
import java.util.Objects;
import org.apache.flink.api.common.state.MapState;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputDeserializer;
-import org.apache.flink.core.memory.DataOutputSerializer;
import org.apache.flink.statefun.flink.core.generated.MultiplexedStateKey;
import org.apache.flink.statefun.sdk.state.Accessor;
@@ -75,29 +72,4 @@ final class MultiplexedMapStateAccessor<T> implements Accessor<T> {
throw new RuntimeException(e);
}
}
-
- private static final class RawSerializer<T> {
- private final TypeSerializer<T> delegate;
- private final DataOutputSerializer output;
- private final DataInputDeserializer input;
-
- RawSerializer(TypeSerializer<T> delegate) {
- this.delegate = Objects.requireNonNull(delegate);
- this.output = new DataOutputSerializer(32);
- this.input = new DataInputDeserializer();
- }
-
- byte[] serialize(T value) throws IOException {
- output.clear();
- delegate.serialize(value, output);
- return output.getCopyOfBuffer(); // TODO: consider avoiding buffer copying
- }
-
- T deserialize(byte[] bytes) throws IOException {
- input.setBuffer(bytes);
- final T value = delegate.deserialize(input);
- input.releaseArrays();
- return value;
- }
- }
}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedTableStateAccessor.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedTableStateAccessor.java
new file mode 100644
index 0000000..9f466e1
--- /dev/null
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedTableStateAccessor.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.statefun.flink.core.state;
+
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.util.Objects;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.statefun.flink.core.cache.SingleThreadedLruCache;
+import org.apache.flink.statefun.flink.core.generated.MultiplexedStateKey;
+import org.apache.flink.statefun.sdk.state.TableAccessor;
+
+final class MultiplexedTableStateAccessor<K, V> implements TableAccessor<K, V> {
+
+ private static final int COMMON_KEY_CACHE_SIZE = 128;
+
+ private final MapState<MultiplexedStateKey, byte[]> mapStateHandle;
+ private final MultiplexedStateKey accessorMapKeyPrefix;
+ private final RawSerializer<K> keySerializer;
+ private final RawSerializer<V> valueSerializer;
+
+ private final SingleThreadedLruCache<K, MultiplexedStateKey> commonKeysCache =
+ new SingleThreadedLruCache<>(COMMON_KEY_CACHE_SIZE);
+
+ MultiplexedTableStateAccessor(
+ MapState<MultiplexedStateKey, byte[]> handle,
+ MultiplexedStateKey accessorMapKeyPrefix,
+ TypeSerializer<K> subKeySerializer,
+ TypeSerializer<V> subValueSerializer) {
+ this.mapStateHandle = Objects.requireNonNull(handle);
+ this.keySerializer = new RawSerializer<>(subKeySerializer);
+ this.valueSerializer = new RawSerializer<>(subValueSerializer);
+ this.accessorMapKeyPrefix = Objects.requireNonNull(accessorMapKeyPrefix);
+ }
+
+ @Override
+ public void set(K key, V value) {
+ try {
+ MultiplexedStateKey keyBytes = stateKey(key);
+ if (value == null) {
+ mapStateHandle.remove(keyBytes);
+ } else {
+ byte[] bytes = valueSerializer.serialize(value);
+ mapStateHandle.put(keyBytes, bytes);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public V get(K userKey) {
+ try {
+ final MultiplexedStateKey stateKey = stateKey(userKey);
+ final byte[] bytes = mapStateHandle.get(stateKey);
+ if (bytes == null) {
+ return null;
+ }
+ return valueSerializer.deserialize(bytes);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void remove(K userKey) {
+ try {
+ mapStateHandle.remove(stateKey(userKey));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Nonnull
+ private MultiplexedStateKey stateKey(final K userKey) {
+ Objects.requireNonNull(userKey, "Key can not be NULL");
+ @Nullable MultiplexedStateKey stateKey = commonKeysCache.get(userKey);
+ if (stateKey != null) {
+ return stateKey;
+ }
+ try {
+ commonKeysCache.put(userKey, stateKey = computeStateKeyFromUserKey(userKey));
+ return stateKey;
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to serialize the key " + userKey, e);
+ }
+ }
+
+ private MultiplexedStateKey computeStateKeyFromUserKey(K userKey) throws IOException {
+ byte[] userKeyBytes = keySerializer.serialize(userKey);
+ ByteString userKeyByteString = ByteString.copyFrom(userKeyBytes);
+ return accessorMapKeyPrefix.toBuilder().addUserKeys(userKeyByteString).build();
+ }
+}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/RawSerializer.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/RawSerializer.java
new file mode 100644
index 0000000..af0d654
--- /dev/null
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/RawSerializer.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.statefun.flink.core.state;
+
+import java.io.IOException;
+import java.util.Objects;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataOutputSerializer;
+
+final class RawSerializer<T> {
+ private final TypeSerializer<T> delegate;
+ private final DataOutputSerializer output;
+ private final DataInputDeserializer input;
+
+ RawSerializer(TypeSerializer<T> delegate) {
+ this.delegate = Objects.requireNonNull(delegate);
+ this.output = new DataOutputSerializer(32);
+ this.input = new DataInputDeserializer();
+ }
+
+ byte[] serialize(T value) throws IOException {
+ output.clear();
+ delegate.serialize(value, output);
+ return output.getCopyOfBuffer(); // TODO: consider avoiding buffer copying
+ }
+
+ T deserialize(byte[] bytes) throws IOException {
+ input.setBuffer(bytes);
+ final T value = delegate.deserialize(input);
+ input.releaseArrays();
+ return value;
+ }
+}
[flink-statefun] 02/09: [FLINK-15954] Add PersistedTable to bound
state
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit 45f4d17a6d955470b28aad48d30aa8178c38a953
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Fri Feb 7 17:23:30 2020 +0100
[FLINK-15954] Add PersistedTable to bound state
This commit adds the PersistedTable and also refactoros this class
to use a builder.
---
.../statefun/flink/core/state/BoundState.java | 44 ++++++++++++++++++++--
1 file changed, 40 insertions(+), 4 deletions(-)
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/BoundState.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/BoundState.java
index 6a99de2..f7b131a 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/BoundState.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/BoundState.java
@@ -19,18 +19,54 @@ package org.apache.flink.statefun.flink.core.state;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
+import org.apache.flink.statefun.sdk.state.PersistedTable;
import org.apache.flink.statefun.sdk.state.PersistedValue;
public class BoundState {
- private final List<PersistedValue<Object>> persistedValues;
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ private final List<PersistedValue<?>> persistedValues;
+ private final List<PersistedTable<?, ?>> persistedTables;
- BoundState(List<PersistedValue<Object>> persistedValues) {
- this.persistedValues = new ArrayList<>(persistedValues);
+ private BoundState(
+ List<PersistedValue<?>> persistedValues, List<PersistedTable<?, ?>> persistedTables) {
+ this.persistedValues = Objects.requireNonNull(persistedValues);
+ this.persistedTables = Objects.requireNonNull(persistedTables);
}
@SuppressWarnings("unused")
- public List<PersistedValue<Object>> persistedValues() {
+ public List<PersistedValue<?>> persistedValues() {
return persistedValues;
}
+
+ @SuppressWarnings("unused")
+ public List<PersistedTable<?, ?>> getPersistedTables() {
+ return persistedTables;
+ }
+
+ @SuppressWarnings("UnusedReturnValue")
+ public static final class Builder {
+ private List<PersistedValue<?>> persistedValues = new ArrayList<>();
+ private List<PersistedTable<?, ?>> persistedTables = new ArrayList<>();
+
+ private Builder() {}
+
+ public Builder withPersistedValue(PersistedValue<?> persistedValue) {
+ this.persistedValues.add(persistedValue);
+ return this;
+ }
+
+ public Builder withPersistedTable(PersistedTable<?, ?> persistedTable) {
+ this.persistedTables.add(persistedTable);
+ return this;
+ }
+
+ public BoundState build() {
+ return new BoundState(persistedValues, persistedTables);
+ }
+ }
}
[flink-statefun] 07/09: [FLINK-15955] Add an HttpFunctionSpec
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit ae9b5dbd72e84d514b8c2edc54a9176a3d51b07b
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Fri Feb 7 18:18:40 2020 +0100
[FLINK-15955] Add an HttpFunctionSpec
---
.../flink/core/jsonmodule/FunctionSpec.java | 32 +++++++++++++
.../flink/core/jsonmodule/HttpFunctionSpec.java | 53 ++++++++++++++++++++++
.../statefun/flink/core/jsonmodule/Pointers.java | 4 ++
3 files changed, 89 insertions(+)
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/FunctionSpec.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/FunctionSpec.java
new file mode 100644
index 0000000..097c3cf
--- /dev/null
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/FunctionSpec.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.statefun.flink.core.jsonmodule;
+
+import org.apache.flink.statefun.sdk.FunctionType;
+
+public interface FunctionSpec {
+
+ FunctionType functionType();
+
+ Kind kind();
+
+ enum Kind {
+ HTTP,
+ GRPC
+ }
+}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/HttpFunctionSpec.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/HttpFunctionSpec.java
new file mode 100644
index 0000000..f37358f
--- /dev/null
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/HttpFunctionSpec.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.statefun.flink.core.jsonmodule;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Objects;
+import org.apache.flink.statefun.sdk.FunctionType;
+
+public final class HttpFunctionSpec implements FunctionSpec {
+ private final FunctionType functionType;
+ private final URI endpoint;
+ private final List<String> states;
+
+ public HttpFunctionSpec(FunctionType functionType, URI endpoint, List<String> states) {
+ this.functionType = Objects.requireNonNull(functionType);
+ this.endpoint = Objects.requireNonNull(endpoint);
+ this.states = Objects.requireNonNull(states);
+ }
+
+ @Override
+ public FunctionType functionType() {
+ return functionType;
+ }
+
+ @Override
+ public Kind kind() {
+ return Kind.HTTP;
+ }
+
+ public URI endpoint() {
+ return endpoint;
+ }
+
+ public List<String> states() {
+ return states;
+ }
+}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/Pointers.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/Pointers.java
index 9745036..fb25ac1 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/Pointers.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/jsonmodule/Pointers.java
@@ -39,9 +39,13 @@ public final class Pointers {
// -------------------------------------------------------------------------------------
public static final class Functions {
+ public static final JsonPointer META_KIND = JsonPointer.compile("/function/meta/kind");
public static final JsonPointer META_TYPE = JsonPointer.compile("/function/meta/type");
public static final JsonPointer FUNCTION_HOSTNAME = JsonPointer.compile("/function/spec/host");
+ public static final JsonPointer FUNCTION_ENDPOINT =
+ JsonPointer.compile("/function/spec/endpoint");
public static final JsonPointer FUNCTION_PORT = JsonPointer.compile("/function/spec/port");
+ public static final JsonPointer FUNCTION_STATES = JsonPointer.compile("/function/spec/states");
}
// -------------------------------------------------------------------------------------
[flink-statefun] 06/09: [FLINK-15954] Add PersistedTable support in
statefun runtime
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit 65ed306e33bde474811d10dfc8d93d715a4ee2f2
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Fri Feb 7 17:43:10 2020 +0100
[FLINK-15954] Add PersistedTable support in statefun runtime
This closes #17.
---
.../flink/statefun/flink/core/state/FlinkState.java | 16 ++++++++++++++++
.../statefun/flink/core/state/MultiplexedState.java | 15 +++++++++++++++
.../apache/flink/statefun/flink/core/state/State.java | 5 +++++
3 files changed, 36 insertions(+)
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/FlinkState.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/FlinkState.java
index dfc5bf6..dfebc8b 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/FlinkState.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/FlinkState.java
@@ -19,6 +19,8 @@ package org.apache.flink.statefun.flink.core.state;
import java.util.Objects;
import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -30,7 +32,9 @@ import org.apache.flink.statefun.flink.core.types.DynamicallyRegisteredTypes;
import org.apache.flink.statefun.sdk.Address;
import org.apache.flink.statefun.sdk.FunctionType;
import org.apache.flink.statefun.sdk.state.Accessor;
+import org.apache.flink.statefun.sdk.state.PersistedTable;
import org.apache.flink.statefun.sdk.state.PersistedValue;
+import org.apache.flink.statefun.sdk.state.TableAccessor;
public final class FlinkState implements State {
@@ -60,6 +64,18 @@ public final class FlinkState implements State {
}
@Override
+ public <K, V> TableAccessor<K, V> createFlinkStateTableAccessor(
+ FunctionType functionType, PersistedTable<K, V> persistedTable) {
+ MapState<K, V> handle =
+ runtimeContext.getMapState(
+ new MapStateDescriptor<>(
+ flinkStateName(functionType, persistedTable.name()),
+ persistedTable.keyType(),
+ persistedTable.valueType()));
+ return new FlinkTableAccessor<>(handle);
+ }
+
+ @Override
public void setCurrentKey(Address address) {
keyedStateBackend.setCurrentKey(KeyBy.apply(address));
}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedState.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedState.java
index 9d9c09b..8ccb7c0 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedState.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/MultiplexedState.java
@@ -35,7 +35,9 @@ import org.apache.flink.statefun.flink.core.types.DynamicallyRegisteredTypes;
import org.apache.flink.statefun.sdk.Address;
import org.apache.flink.statefun.sdk.FunctionType;
import org.apache.flink.statefun.sdk.state.Accessor;
+import org.apache.flink.statefun.sdk.state.PersistedTable;
import org.apache.flink.statefun.sdk.state.PersistedValue;
+import org.apache.flink.statefun.sdk.state.TableAccessor;
public final class MultiplexedState implements State {
@@ -66,6 +68,19 @@ public final class MultiplexedState implements State {
}
@Override
+ public <K, V> TableAccessor<K, V> createFlinkStateTableAccessor(
+ FunctionType functionType, PersistedTable<K, V> persistedTable) {
+ final MultiplexedStateKey uniqueSubKeyPrefix =
+ multiplexedSubstateKey(functionType, persistedTable.name());
+ final TypeSerializer<K> keySerializer =
+ types.registerType(persistedTable.keyType()).createSerializer(executionConfiguration);
+ final TypeSerializer<V> valueSerializer =
+ types.registerType(persistedTable.valueType()).createSerializer(executionConfiguration);
+ return new MultiplexedTableStateAccessor<>(
+ sharedMapStateHandle, uniqueSubKeyPrefix, keySerializer, valueSerializer);
+ }
+
+ @Override
public void setCurrentKey(Address address) {
keyedStateBackend.setCurrentKey(KeyBy.apply(address));
}
diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/State.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/State.java
index 64fffb5..9f1a83a 100644
--- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/State.java
+++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/state/State.java
@@ -20,12 +20,17 @@ package org.apache.flink.statefun.flink.core.state;
import org.apache.flink.statefun.sdk.Address;
import org.apache.flink.statefun.sdk.FunctionType;
import org.apache.flink.statefun.sdk.state.Accessor;
+import org.apache.flink.statefun.sdk.state.PersistedTable;
import org.apache.flink.statefun.sdk.state.PersistedValue;
+import org.apache.flink.statefun.sdk.state.TableAccessor;
public interface State {
<T> Accessor<T> createFlinkStateAccessor(
FunctionType functionType, PersistedValue<T> persistedValue);
+ <K, V> TableAccessor<K, V> createFlinkStateTableAccessor(
+ FunctionType functionType, PersistedTable<K, V> persistedTable);
+
void setCurrentKey(Address address);
}
[flink-statefun] 01/09: [FLINK-15954] Add PersistedTable SDK classes
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit 840b005bcf5dcd493519b3c3befb2233dd4f524c
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Fri Feb 7 15:24:11 2020 +0100
[FLINK-15954] Add PersistedTable SDK classes
---
.../flink/statefun/sdk/state/PersistedTable.java | 148 +++++++++++++++++++++
.../flink/statefun/sdk/state/TableAccessor.java | 27 ++++
2 files changed, 175 insertions(+)
diff --git a/statefun-sdk/src/main/java/org/apache/flink/statefun/sdk/state/PersistedTable.java b/statefun-sdk/src/main/java/org/apache/flink/statefun/sdk/state/PersistedTable.java
new file mode 100644
index 0000000..f54166f
--- /dev/null
+++ b/statefun-sdk/src/main/java/org/apache/flink/statefun/sdk/state/PersistedTable.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.statefun.sdk.state;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.flink.statefun.sdk.StatefulFunction;
+import org.apache.flink.statefun.sdk.annotations.ForRuntime;
+import org.apache.flink.statefun.sdk.annotations.Persisted;
+
+/**
+ * A {@link PersistedTable} is a table (collection of keys and values) registered within {@link
+ * StatefulFunction}s and is persisted and maintained by the system for fault-tolerance.
+ *
+ * <p>Created persisted tables must be registered by using the {@link Persisted} annotation. Please
+ * see the class-level Javadoc of {@link StatefulFunction} for an example on how to do that.
+ *
+ * @see StatefulFunction
+ * @param <K> type of the key - Please note that the key have a meaningful {@link #hashCode()} and
+ * {@link #equals(Object)} implemented.
+ * @param <V> type of the value.
+ */
+public final class PersistedTable<K, V> {
+ private final String name;
+ private final Class<K> keyType;
+ private final Class<V> valueType;
+ private TableAccessor<K, V> accessor;
+
+ private PersistedTable(
+ String name, Class<K> keyType, Class<V> valueType, TableAccessor<K, V> accessor) {
+ this.name = Objects.requireNonNull(name);
+ this.keyType = Objects.requireNonNull(keyType);
+ this.valueType = Objects.requireNonNull(valueType);
+ this.accessor = Objects.requireNonNull(accessor);
+ }
+
+ /**
+ * Creates a {@link PersistedTable} instance that may be used to access persisted state managed by
+ * the system. Access to the persisted table is identified by an unique name, type of the key, and
+ * type of the value. These may not change across multiple executions of the application.
+ *
+ * @param name the unique name of the persisted state.
+ * @param keyType the type of the state keys of this {@code PersistedTable}.
+ * @param valueType the type of the state values of this {@code PersistedTale}.
+ * @param <K> the type of the state keys.
+ * @param <V> the type of the state values.
+ * @return a {@code PersistedTable} instance.
+ */
+ public static <K, V> PersistedTable<K, V> of(String name, Class<K> keyType, Class<V> valueType) {
+ return new PersistedTable<>(name, keyType, valueType, new NonFaultTolerantAccessor<>());
+ }
+
+ /**
+ * Returns the unique name of the persisted table.
+ *
+ * @return unique name of the persisted table.
+ */
+ public String name() {
+ return name;
+ }
+
+ /**
+ * Returns the type of the persisted tables keys.
+ *
+ * @return the type of the persisted tables keys.
+ */
+ public Class<K> keyType() {
+ return keyType;
+ }
+
+ /**
+ * Returns the type of the persisted tables values.
+ *
+ * @return the type of the persisted tables values.
+ */
+ public Class<V> valueType() {
+ return valueType;
+ }
+
+ /**
+ * Returns a persisted table's value.
+ *
+ * @return the persisted table value associated with {@code key}.
+ */
+ public V get(K key) {
+ return accessor.get(key);
+ }
+
+ /**
+ * Updates the persisted table.
+ *
+ * @param key the to associate the value with.
+ * @param value the new value.
+ */
+ public void set(K key, V value) {
+ accessor.set(key, value);
+ }
+
+ /**
+ * Removes the value associated with {@code key}.
+ *
+ * @param key the key to remove.
+ */
+ public void remove(K key) {
+ accessor.remove(key);
+ }
+
+ @ForRuntime
+ void setAccessor(TableAccessor<K, V> newAccessor) {
+ Objects.requireNonNull(newAccessor);
+ this.accessor = newAccessor;
+ }
+
+ private static final class NonFaultTolerantAccessor<K, V> implements TableAccessor<K, V> {
+ private final Map<K, V> map = new HashMap<>();
+
+ @Override
+ public void set(K key, V value) {
+ map.put(key, value);
+ }
+
+ @Override
+ public V get(K key) {
+ return map.get(key);
+ }
+
+ @Override
+ public void remove(K key) {
+ map.remove(key);
+ }
+ }
+}
diff --git a/statefun-sdk/src/main/java/org/apache/flink/statefun/sdk/state/TableAccessor.java b/statefun-sdk/src/main/java/org/apache/flink/statefun/sdk/state/TableAccessor.java
new file mode 100644
index 0000000..af68972
--- /dev/null
+++ b/statefun-sdk/src/main/java/org/apache/flink/statefun/sdk/state/TableAccessor.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.statefun.sdk.state;
+
+public interface TableAccessor<K, V> {
+
+ void set(K key, V value);
+
+ V get(K key);
+
+ void remove(K key);
+}