You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sr...@apache.org on 2019/02/22 19:16:48 UTC
[flink] branch master updated: [FLINK-10043] [State Backends]
Refactor RocksDBKeyedStateBackend object
construction/initialization/restore code
This is an automated email from the ASF dual-hosted git repository.
srichter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 3bf06b9 [FLINK-10043] [State Backends] Refactor RocksDBKeyedStateBackend object construction/initialization/restore code
3bf06b9 is described below
commit 3bf06b9dd3dc66dcfdeccaaa591ace41ea51e343
Author: Yu Li <li...@apache.org>
AuthorDate: Sat Feb 23 03:16:41 2019 +0800
[FLINK-10043] [State Backends] Refactor RocksDBKeyedStateBackend object construction/initialization/restore code
This closes #7674.
---
.../flink/streaming/tests/StubStateBackend.java | 41 +-
.../KVStateRequestSerializerRocksDBTest.java | 43 +-
.../network/KvStateServerHandlerTest.java | 3 +-
.../runtime/state/AbstractKeyedStateBackend.java | 37 +-
.../state/AbstractKeyedStateBackendBuilder.java | 67 ++
.../flink/runtime/state/AbstractStateBackend.java | 6 +-
.../runtime/state/BackendBuildingException.java | 30 +
.../apache/flink/runtime/state/StateBackend.java | 44 +-
.../flink/runtime/state/StateBackendBuilder.java | 29 +
.../runtime/state/filesystem/FsStateBackend.java | 5 +-
.../runtime/state/heap/HeapKeyedStateBackend.java | 3 +-
.../runtime/state/memory/MemoryStateBackend.java | 21 +-
.../CheckpointSettingsSerializableTest.java | 5 +-
.../state/StateBackendMigrationTestBase.java | 7 +-
.../flink/runtime/state/StateBackendTestBase.java | 19 +-
.../runtime/state/ttl/StateBackendTestContext.java | 16 +-
.../flink/runtime/state/ttl/TtlStateTestBase.java | 6 +-
.../state/ttl/mock/MockKeyedStateBackend.java | 3 +-
.../runtime/state/ttl/mock/MockStateBackend.java | 5 +-
.../state/RocksDBIncrementalCheckpointUtils.java | 8 +-
.../streaming/state/RocksDBKeyedStateBackend.java | 1223 ++------------------
.../state/RocksDBKeyedStateBackendBuilder.java | 511 ++++++++
.../contrib/streaming/state/RocksDBMapState.java | 4 +-
.../state/RocksDBNativeMetricMonitor.java | 6 +-
.../streaming/state/RocksDBOperationUtils.java | 161 +++
.../state/RocksDBPriorityQueueSetFactory.java | 188 +++
.../streaming/state/RocksDBStateBackend.java | 84 +-
.../restore/AbstractRocksDBRestoreOperation.java | 222 ++++
.../state/restore/RocksDBFullRestoreOperation.java | 237 ++++
.../RocksDBIncrementalRestoreOperation.java | 523 +++++++++
.../RocksDBIncrementalRestorePrepareResult.java | 46 +
.../state/restore/RocksDBNoneRestoreOperation.java | 87 ++
.../state/restore/RocksDBRestoreOperation.java | 29 +
.../state/restore/RocksDBRestoreResult.java | 82 ++
.../snapshot/RocksDBSnapshotStrategyBase.java | 2 +-
.../streaming/state/RocksDBAsyncSnapshotTest.java | 3 -
.../state/RocksDBNativeMetricMonitorTest.java | 12 +-
.../state/RocksDBRocksStateKeysIteratorTest.java | 3 +-
.../state/RocksDBStateBackendConfigTest.java | 5 +-
.../streaming/state/RocksDBStateBackendTest.java | 106 +-
...RocksKeyGroupsRocksSingleStateIteratorTest.java | 2 +-
.../state/benchmark/RocksDBPerformanceTest.java | 3 +-
.../api/operators/BackendRestorerProcedure.java | 9 +-
.../operators/StreamTaskStateInitializerImpl.java | 7 +-
.../operators/BackendRestorerProcedureTest.java | 9 +-
.../StreamTaskStateInitializerImplTest.java | 4 +-
.../runtime/tasks/StreamTaskTerminationTest.java | 5 +-
.../runtime/tasks/TestSpyWrapperStateBackend.java | 7 +-
.../test/streaming/runtime/StateBackendITCase.java | 5 +-
49 files changed, 2662 insertions(+), 1321 deletions(-)
diff --git a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/StubStateBackend.java b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/StubStateBackend.java
index b93fa36..17c51d8 100644
--- a/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/StubStateBackend.java
+++ b/flink-end-to-end-tests/flink-stream-state-ttl-test/src/main/java/org/apache/flink/streaming/tests/StubStateBackend.java
@@ -27,11 +27,13 @@ import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.StateBackend;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import java.io.IOException;
+import java.util.Collection;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -64,27 +66,28 @@ final class StubStateBackend implements StateBackend {
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
- Environment env,
- JobID jobID,
- String operatorIdentifier,
- TypeSerializer<K> keySerializer,
- int numberOfKeyGroups,
- KeyGroupRange keyGroupRange,
- TaskKvStateRegistry kvStateRegistry,
- TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) throws Exception {
+ Environment env,
+ JobID jobID,
+ String operatorIdentifier,
+ TypeSerializer<K> keySerializer,
+ int numberOfKeyGroups,
+ KeyGroupRange keyGroupRange,
+ TaskKvStateRegistry kvStateRegistry,
+ TtlTimeProvider ttlTimeProvider,
+ MetricGroup metricGroup,
+ Collection<KeyedStateHandle> stateHandles) throws Exception {
return backend.createKeyedStateBackend(
- env,
- jobID,
- operatorIdentifier,
- keySerializer,
- numberOfKeyGroups,
- keyGroupRange,
- kvStateRegistry,
- this.ttlTimeProvider,
- metricGroup
- );
+ env,
+ jobID,
+ operatorIdentifier,
+ keySerializer,
+ numberOfKeyGroups,
+ keyGroupRange,
+ kvStateRegistry,
+ this.ttlTimeProvider,
+ metricGroup,
+ stateHandles);
}
@Override
diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KVStateRequestSerializerRocksDBTest.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KVStateRequestSerializerRocksDBTest.java
index 3c4d224..de86340 100644
--- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KVStateRequestSerializerRocksDBTest.java
+++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KVStateRequestSerializerRocksDBTest.java
@@ -25,7 +25,7 @@ import org.apache.flink.api.common.typeutils.base.LongSerializer;
import org.apache.flink.api.common.typeutils.base.StringSerializer;
import org.apache.flink.contrib.streaming.state.PredefinedOptions;
import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend;
-import org.apache.flink.contrib.streaming.state.RocksDBNativeMetricOptions;
+import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder;
import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.queryablestate.client.VoidNamespace;
@@ -40,9 +40,10 @@ import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import org.rocksdb.ColumnFamilyOptions;
import org.rocksdb.DBOptions;
+import java.util.Collections;
+
import static org.mockito.Mockito.mock;
/**
@@ -68,29 +69,26 @@ public final class KVStateRequestSerializerRocksDBTest {
// objects for RocksDB state list serialisation
DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
dbOptions.setCreateIfMissing(true);
- ColumnFamilyOptions columnFamilyOptions = PredefinedOptions.DEFAULT.createColumnOptions();
+ ExecutionConfig executionConfig = new ExecutionConfig();
final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
- new RocksDBKeyedStateBackend<>(
+ new RocksDBKeyedStateBackendBuilder<>(
"no-op",
ClassLoader.getSystemClassLoader(),
temporaryFolder.getRoot(),
dbOptions,
- stateName -> columnFamilyOptions,
+ stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
mock(TaskKvStateRegistry.class),
LongSerializer.INSTANCE,
1,
new KeyGroupRange(0, 0),
- new ExecutionConfig(),
- false,
- 1,
+ executionConfig,
TestLocalRecoveryConfig.disabled(),
RocksDBStateBackend.PriorityQueueStateType.HEAP,
TtlTimeProvider.DEFAULT,
- false,
- new RocksDBNativeMetricOptions(),
- new UnregisteredMetricsGroup()
- );
- longHeapKeyedStateBackend.restore(null);
+ new UnregisteredMetricsGroup(),
+ Collections.emptyList(),
+ RocksDBStateBackend.getCompressionDecorator(executionConfig)
+ ).build();
longHeapKeyedStateBackend.setCurrentKey(key);
final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
@@ -114,29 +112,26 @@ public final class KVStateRequestSerializerRocksDBTest {
// objects for RocksDB state list serialisation
DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
dbOptions.setCreateIfMissing(true);
- ColumnFamilyOptions columnFamilyOptions = PredefinedOptions.DEFAULT.createColumnOptions();
+ ExecutionConfig executionConfig = new ExecutionConfig();
final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
- new RocksDBKeyedStateBackend<Long>(
+ new RocksDBKeyedStateBackendBuilder<>(
"no-op",
ClassLoader.getSystemClassLoader(),
temporaryFolder.getRoot(),
dbOptions,
- stateName -> columnFamilyOptions,
+ stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
mock(TaskKvStateRegistry.class),
LongSerializer.INSTANCE,
1,
new KeyGroupRange(0, 0),
- new ExecutionConfig(),
- false,
- 1,
+ executionConfig,
TestLocalRecoveryConfig.disabled(),
RocksDBStateBackend.PriorityQueueStateType.HEAP,
TtlTimeProvider.DEFAULT,
- false,
- new RocksDBNativeMetricOptions(),
- new UnregisteredMetricsGroup()
- );
- longHeapKeyedStateBackend.restore(null);
+ new UnregisteredMetricsGroup(),
+ Collections.emptyList(),
+ RocksDBStateBackend.getCompressionDecorator(executionConfig)
+ ).build();
longHeapKeyedStateBackend.setCurrentKey(key);
final InternalMapState<Long, VoidNamespace, Long, String> mapState =
diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KvStateServerHandlerTest.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KvStateServerHandlerTest.java
index 5ce889c..7edb60e 100644
--- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KvStateServerHandlerTest.java
+++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KvStateServerHandlerTest.java
@@ -770,6 +770,7 @@ public class KvStateServerHandlerTest extends TestLogger {
new KeyGroupRange(0, 0),
registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()),
TtlTimeProvider.DEFAULT,
- new UnregisteredMetricsGroup());
+ new UnregisteredMetricsGroup(),
+ null);
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
index 03e36bb..98e21ff 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
@@ -57,7 +57,7 @@ public abstract class AbstractKeyedStateBackend<K> implements
CheckpointListener {
/** {@link StateSerializerProvider} for our key serializer. */
- private final StateSerializerProvider<K> keySerializerProvider;
+ protected final StateSerializerProvider<K> keySerializerProvider;
/** The currently active key. */
private K currentKey;
@@ -105,25 +105,48 @@ public abstract class AbstractKeyedStateBackend<K> implements
int numberOfKeyGroups,
KeyGroupRange keyGroupRange,
ExecutionConfig executionConfig,
- TtlTimeProvider ttlTimeProvider) {
+ TtlTimeProvider ttlTimeProvider,
+ CloseableRegistry cancelStreamRegistry) {
+ this(
+ kvStateRegistry,
+ StateSerializerProvider.fromNewRegisteredSerializer(keySerializer),
+ userCodeClassLoader,
+ numberOfKeyGroups,
+ keyGroupRange,
+ executionConfig,
+ ttlTimeProvider,
+ cancelStreamRegistry,
+ determineStreamCompression(executionConfig)
+ );
+ }
+ public AbstractKeyedStateBackend(
+ TaskKvStateRegistry kvStateRegistry,
+ StateSerializerProvider<K> keySerializerProvider,
+ ClassLoader userCodeClassLoader,
+ int numberOfKeyGroups,
+ KeyGroupRange keyGroupRange,
+ ExecutionConfig executionConfig,
+ TtlTimeProvider ttlTimeProvider,
+ CloseableRegistry cancelStreamRegistry,
+ StreamCompressionDecorator keyGroupCompressionDecorator) {
Preconditions.checkArgument(numberOfKeyGroups >= 1, "NumberOfKeyGroups must be a positive number");
Preconditions.checkArgument(numberOfKeyGroups >= keyGroupRange.getNumberOfKeyGroups(), "The total number of key groups must be at least the number in the key group range assigned to this backend");
this.kvStateRegistry = kvStateRegistry;
- this.keySerializerProvider = StateSerializerProvider.fromNewRegisteredSerializer(keySerializer);
+ this.keySerializerProvider = keySerializerProvider;
this.numberOfKeyGroups = numberOfKeyGroups;
this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange);
- this.cancelStreamRegistry = new CloseableRegistry();
+ this.cancelStreamRegistry = cancelStreamRegistry;
this.keyValueStatesByName = new HashMap<>();
this.executionConfig = executionConfig;
- this.keyGroupCompressionDecorator = determineStreamCompression(executionConfig);
+ this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
this.ttlTimeProvider = Preconditions.checkNotNull(ttlTimeProvider);
this.keySelectionListeners = new HashSet<>();
}
- private StreamCompressionDecorator determineStreamCompression(ExecutionConfig executionConfig) {
+ private static StreamCompressionDecorator determineStreamCompression(ExecutionConfig executionConfig) {
if (executionConfig != null && executionConfig.isUseSnapshotCompression()) {
return SnappyStreamCompressionDecorator.INSTANCE;
} else {
@@ -337,7 +360,7 @@ public abstract class AbstractKeyedStateBackend<K> implements
}
@VisibleForTesting
- StreamCompressionDecorator getKeyGroupCompressionDecorator() {
+ public StreamCompressionDecorator getKeyGroupCompressionDecorator() {
return keyGroupCompressionDecorator;
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackendBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackendBuilder.java
new file mode 100644
index 0000000..a71272e
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackendBuilder.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.runtime.query.TaskKvStateRegistry;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+
+import javax.annotation.Nonnull;
+
+import java.util.Collection;
+
+/**
+ * An abstract base implementation of the {@link StateBackendBuilder} interface.
+ */
+public abstract class AbstractKeyedStateBackendBuilder<K>
+ implements StateBackendBuilder<AbstractKeyedStateBackend, BackendBuildingException> {
+ protected final TaskKvStateRegistry kvStateRegistry;
+ protected final TypeSerializer<K> keySerializer;
+ protected final StateSerializerProvider<K> keySerializerProvider;
+ protected final ClassLoader userCodeClassLoader;
+ protected final int numberOfKeyGroups;
+ protected final KeyGroupRange keyGroupRange;
+ protected final ExecutionConfig executionConfig;
+ protected final TtlTimeProvider ttlTimeProvider;
+ protected final StreamCompressionDecorator keyGroupCompressionDecorator;
+ protected final Collection<KeyedStateHandle> restoreStateHandles;
+
+ public AbstractKeyedStateBackendBuilder(
+ TaskKvStateRegistry kvStateRegistry,
+ TypeSerializer<K> keySerializer,
+ ClassLoader userCodeClassLoader,
+ int numberOfKeyGroups,
+ KeyGroupRange keyGroupRange,
+ ExecutionConfig executionConfig,
+ TtlTimeProvider ttlTimeProvider,
+ @Nonnull Collection<KeyedStateHandle> stateHandles,
+ StreamCompressionDecorator keyGroupCompressionDecorator) {
+ this.kvStateRegistry = kvStateRegistry;
+ this.keySerializer = keySerializer;
+ this.keySerializerProvider = StateSerializerProvider.fromNewRegisteredSerializer(keySerializer);
+ this.userCodeClassLoader = userCodeClassLoader;
+ this.numberOfKeyGroups = numberOfKeyGroups;
+ this.keyGroupRange = keyGroupRange;
+ this.executionConfig = executionConfig;
+ this.ttlTimeProvider = ttlTimeProvider;
+ this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
+ this.restoreStateHandles = stateHandles;
+ }
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java
index 6c72acd..845a865 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java
@@ -26,7 +26,10 @@ import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import javax.annotation.Nonnull;
+
import java.io.IOException;
+import java.util.Collection;
/**
* An abstract base implementation of the {@link StateBackend} interface.
@@ -52,7 +55,8 @@ public abstract class AbstractStateBackend implements StateBackend, java.io.Seri
KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry,
TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) throws IOException;
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> stateHandles) throws IOException;
@Override
public abstract OperatorStateBackend createOperatorStateBackend(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/BackendBuildingException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/BackendBuildingException.java
new file mode 100644
index 0000000..109b3ab
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/BackendBuildingException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import java.io.IOException;
+
+/**
+ * Exceptions which indicate that a state backend building has failed.
+ */
+public class BackendBuildingException extends IOException {
+ public BackendBuildingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackend.java
index 4235f76..25ef8d2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackend.java
@@ -27,7 +27,11 @@ import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import javax.annotation.Nonnull;
+
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
/**
* A <b>State Backend</b> defines how the state of a streaming application is stored and
@@ -181,7 +185,42 @@ public interface StateBackend extends java.io.Serializable {
keyGroupRange,
kvStateRegistry,
ttlTimeProvider,
- new UnregisteredMetricsGroup());
+ new UnregisteredMetricsGroup(),
+ Collections.emptyList());
+ }
+
+ /**
+ * Creates a new {@link AbstractKeyedStateBackend} that is responsible for holding <b>keyed state</b>
+ * and checkpointing it.
+ *
+ * <p><i>Keyed State</i> is state where each value is bound to a key.
+ *
+ * @param <K> The type of the keys by which the state is organized.
+ * @return The Keyed State Backend for the given job, operator, and key group range.
+ * @throws Exception This method may forward all exceptions that occur while instantiating the backend.
+ */
+ default <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
+ Environment env,
+ JobID jobID,
+ String operatorIdentifier,
+ TypeSerializer<K> keySerializer,
+ int numberOfKeyGroups,
+ KeyGroupRange keyGroupRange,
+ TaskKvStateRegistry kvStateRegistry,
+ TtlTimeProvider ttlTimeProvider,
+ Collection<KeyedStateHandle> stateHandles
+ ) throws Exception {
+ return createKeyedStateBackend(
+ env,
+ jobID,
+ operatorIdentifier,
+ keySerializer,
+ numberOfKeyGroups,
+ keyGroupRange,
+ kvStateRegistry,
+ ttlTimeProvider,
+ new UnregisteredMetricsGroup(),
+ stateHandles);
}
/**
@@ -205,7 +244,8 @@ public interface StateBackend extends java.io.Serializable {
KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry,
TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) throws Exception;
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> stateHandles) throws Exception;
/**
* Creates a new {@link OperatorStateBackend} that can be used for storing operator state.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackendBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackendBuilder.java
new file mode 100644
index 0000000..6aacaab
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackendBuilder.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+/**
+ * A mutable builder to build a state backend instance.
+ *
+ * @param <T> The type of the state backend instance.
+ * @param <E> The type of Exceptions thrown in build.
+ */
+public interface StateBackendBuilder<T, E extends Throwable> {
+ T build() throws E;
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java
index bf9a49d..3986ab8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java
@@ -32,6 +32,7 @@ import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.ConfigurableStateBackend;
import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.LocalRecoveryConfig;
import org.apache.flink.runtime.state.OperatorStateBackend;
@@ -48,6 +49,7 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.net.URI;
+import java.util.Collection;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -458,7 +460,8 @@ public class FsStateBackend extends AbstractFileStateBackend implements Configur
KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry,
TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) {
+ MetricGroup metricGroup,
+ Collection<KeyedStateHandle> stateHandles) {
TaskStateManager taskStateManager = env.getTaskStateManager();
LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index 56374fe..12d7b26 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -31,6 +31,7 @@ import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
@@ -151,7 +152,7 @@ public class HeapKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
TtlTimeProvider ttlTimeProvider) {
super(kvStateRegistry, keySerializer, userCodeClassLoader,
- numberOfKeyGroups, keyGroupRange, executionConfig, ttlTimeProvider);
+ numberOfKeyGroups, keyGroupRange, executionConfig, ttlTimeProvider, new CloseableRegistry());
this.registeredKVStates = new HashMap<>();
this.registeredPQStates = new HashMap<>();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java
index eaa1242..ffb7d7a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java
@@ -31,6 +31,7 @@ import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.ConfigurableStateBackend;
import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.TaskStateManager;
@@ -43,6 +44,7 @@ import org.apache.flink.util.TernaryBoolean;
import javax.annotation.Nullable;
import java.io.IOException;
+import java.util.Collection;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -305,15 +307,16 @@ public class MemoryStateBackend extends AbstractFileStateBackend implements Conf
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
- Environment env,
- JobID jobID,
- String operatorIdentifier,
- TypeSerializer<K> keySerializer,
- int numberOfKeyGroups,
- KeyGroupRange keyGroupRange,
- TaskKvStateRegistry kvStateRegistry,
- TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) {
+ Environment env,
+ JobID jobID,
+ String operatorIdentifier,
+ TypeSerializer<K> keySerializer,
+ int numberOfKeyGroups,
+ KeyGroupRange keyGroupRange,
+ TaskKvStateRegistry kvStateRegistry,
+ TtlTimeProvider ttlTimeProvider,
+ MetricGroup metricGroup,
+ Collection<KeyedStateHandle> stateHandles) {
TaskStateManager taskStateManager = env.getTaskStateManager();
HeapPriorityQueueSetFactory priorityQueueSetFactory =
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
index 4b5ae06..a5a2b5e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
@@ -39,6 +39,7 @@ import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.StateBackend;
@@ -53,6 +54,7 @@ import java.io.IOException;
import java.io.Serializable;
import java.net.URL;
import java.net.URLClassLoader;
+import java.util.Collection;
import java.util.Collections;
import static org.junit.Assert.assertEquals;
@@ -172,7 +174,8 @@ public class CheckpointSettingsSerializableTest extends TestLogger {
KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry,
TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) throws Exception {
+ MetricGroup metricGroup,
+ Collection<KeyedStateHandle> stateHandles) throws Exception {
throw new UnsupportedOperationException();
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java
index 3177dd4..13009a7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java
@@ -34,6 +34,7 @@ import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.checkpoint.StateObjectCollection;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.apache.flink.runtime.testutils.statemigration.TestType;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.StateMigrationException;
@@ -999,7 +1000,7 @@ public abstract class StateBackendMigrationTestBase<B extends AbstractStateBacke
numberOfKeyGroups,
keyGroupRange,
env.getTaskKvStateRegistry());
- backend.restore(null);
+ backend.restore(Collections.emptyList());
return backend;
}
@@ -1032,7 +1033,9 @@ public abstract class StateBackendMigrationTestBase<B extends AbstractStateBacke
keySerializer,
numberOfKeyGroups,
keyGroupRange,
- env.getTaskKvStateRegistry());
+ env.getTaskKvStateRegistry()
+ , TtlTimeProvider.DEFAULT,
+ state);
backend.restore(new StateObjectCollection<>(state));
return backend;
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
index a4306fd..b7d8cfc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
@@ -18,6 +18,10 @@
package org.apache.flink.runtime.state;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.functions.AggregateFunction;
import org.apache.flink.api.common.functions.FoldFunction;
@@ -72,17 +76,11 @@ import org.apache.flink.runtime.state.internal.InternalReducingState;
import org.apache.flink.runtime.state.internal.InternalValueState;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.apache.flink.runtime.util.BlockerCheckpointStreamFactory;
+import org.apache.flink.shaded.guava18.com.google.common.base.Joiner;
import org.apache.flink.types.IntValue;
import org.apache.flink.util.IOUtils;
import org.apache.flink.util.StateMigrationException;
import org.apache.flink.util.TestLogger;
-
-import org.apache.flink.shaded.guava18.com.google.common.base.Joiner;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import org.apache.commons.io.output.ByteArrayOutputStream;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -181,7 +179,7 @@ public abstract class StateBackendTestBase<B extends AbstractStateBackend> exten
env.getTaskKvStateRegistry(),
TtlTimeProvider.DEFAULT);
- backend.restore(null);
+ backend.restore(Collections.emptyList());
return backend;
}
@@ -217,7 +215,8 @@ public abstract class StateBackendTestBase<B extends AbstractStateBackend> exten
numberOfKeyGroups,
keyGroupRange,
env.getTaskKvStateRegistry(),
- TtlTimeProvider.DEFAULT);
+ TtlTimeProvider.DEFAULT,
+ state);
backend.restore(new StateObjectCollection<>(state));
@@ -3030,6 +3029,8 @@ public abstract class StateBackendTestBase<B extends AbstractStateBackend> exten
fail("should recognize wrong key serializer");
} catch (StateMigrationException ignored) {
// expected
+ } catch (BackendBuildingException ignored) {
+ assertTrue(ignored.getCause() instanceof StateMigrationException);
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/StateBackendTestContext.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/StateBackendTestContext.java
index b48b9ba..1c4294d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/StateBackendTestContext.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/StateBackendTestContext.java
@@ -42,6 +42,7 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.concurrent.RunnableFuture;
@@ -77,17 +78,24 @@ public abstract class StateBackendTestContext {
}
}
- void createAndRestoreKeyedStateBackend() {
- createAndRestoreKeyedStateBackend(NUMBER_OF_KEY_GROUPS);
+ void createAndRestoreKeyedStateBackend(KeyedStateHandle snapshot) {
+ createAndRestoreKeyedStateBackend(NUMBER_OF_KEY_GROUPS, snapshot);
}
- void createAndRestoreKeyedStateBackend(int numberOfKeyGroups) {
+ void createAndRestoreKeyedStateBackend(int numberOfKeyGroups, KeyedStateHandle snapshot) {
+ Collection<KeyedStateHandle> stateHandles;
+ if (snapshot == null) {
+ stateHandles = Collections.emptyList();
+ } else {
+ stateHandles = new ArrayList<>(1);
+ stateHandles.add(snapshot);
+ }
Environment env = new DummyEnvironment();
try {
disposeKeyedStateBackend();
keyedStateBackend = stateBackend.createKeyedStateBackend(
env, new JobID(), "test", StringSerializer.INSTANCE, numberOfKeyGroups,
- new KeyGroupRange(0, numberOfKeyGroups - 1), env.getTaskKvStateRegistry(), timeProvider);
+ new KeyGroupRange(0, numberOfKeyGroups - 1), env.getTaskKvStateRegistry(), timeProvider, stateHandles);
} catch (Exception e) {
throw new RuntimeException("unexpected", e);
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java
index a2f51e9..443d2a6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java
@@ -128,7 +128,7 @@ public abstract class TtlStateTestBase {
protected <S extends State> StateDescriptor<S, Object> initTest(StateTtlConfig ttlConfig) throws Exception {
this.ttlConfig = ttlConfig;
- sbetc.createAndRestoreKeyedStateBackend();
+ sbetc.createAndRestoreKeyedStateBackend(null);
sbetc.restoreSnapshot(null);
sbetc.setCurrentKey("defaultKey");
StateDescriptor<S, Object> stateDesc = createState();
@@ -154,7 +154,7 @@ public abstract class TtlStateTestBase {
}
private void restoreSnapshot(KeyedStateHandle snapshot, int numberOfKeyGroups) throws Exception {
- sbetc.createAndRestoreKeyedStateBackend(numberOfKeyGroups);
+ sbetc.createAndRestoreKeyedStateBackend(numberOfKeyGroups, snapshot);
sbetc.restoreSnapshot(snapshot);
sbetc.setCurrentKey("defaultKey");
createState();
@@ -433,7 +433,7 @@ public abstract class TtlStateTestBase {
ctx().update(ctx().updateEmpty);
KeyedStateHandle snapshot = sbetc.takeSnapshot();
- sbetc.createAndRestoreKeyedStateBackend();
+ sbetc.createAndRestoreKeyedStateBackend(snapshot);
sbetc.restoreSnapshot(snapshot);
sbetc.setCurrentKey("defaultKey");
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockKeyedStateBackend.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockKeyedStateBackend.java
index b204a00..697bf66 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockKeyedStateBackend.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockKeyedStateBackend.java
@@ -29,6 +29,7 @@ import org.apache.flink.api.common.state.StateDescriptor;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
@@ -100,7 +101,7 @@ public class MockKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
TtlTimeProvider ttlTimeProvider,
MetricGroup operatorMetricGroup) {
super(kvStateRegistry, keySerializer, userCodeClassLoader,
- numberOfKeyGroups, keyGroupRange, executionConfig, ttlTimeProvider);
+ numberOfKeyGroups, keyGroupRange, executionConfig, ttlTimeProvider, new CloseableRegistry());
}
@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockStateBackend.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockStateBackend.java
index e336133..f2bd790 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockStateBackend.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockStateBackend.java
@@ -34,11 +34,13 @@ import org.apache.flink.runtime.state.CheckpointStorageLocationReference;
import org.apache.flink.runtime.state.CheckpointStreamFactory;
import org.apache.flink.runtime.state.CheckpointedStateScope;
import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import javax.annotation.Nullable;
+import java.util.Collection;
/** mack state backend. */
public class MockStateBackend extends AbstractStateBackend {
@@ -120,7 +122,8 @@ public class MockStateBackend extends AbstractStateBackend {
KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry,
TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) {
+ MetricGroup metricGroup,
+ Collection<KeyedStateHandle> stateHandles) {
return new MockKeyedStateBackend<>(
new KvStateRegistry().createTaskRegistry(jobID, new JobVertexID()),
keySerializer,
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalCheckpointUtils.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalCheckpointUtils.java
index 40fac21..6af14f1 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalCheckpointUtils.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalCheckpointUtils.java
@@ -72,7 +72,7 @@ public class RocksDBIncrementalCheckpointUtils {
* @param currentKeyGroupRange the key group range of the db instance.
* @param keyGroupPrefixBytes Number of bytes required to prefix the key groups.
*/
- static void clipDBWithKeyGroupRange(
+ public static void clipDBWithKeyGroupRange(
@Nonnull RocksDB db,
@Nonnull List<ColumnFamilyHandle> columnFamilyHandles,
@Nonnull KeyGroupRange targetKeyGroupRange,
@@ -114,7 +114,7 @@ public class RocksDBIncrementalCheckpointUtils {
byte[] endKeyBytes) throws RocksDBException {
for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) {
- try (RocksIteratorWrapper iteratorWrapper = RocksDBKeyedStateBackend.getRocksIterator(db, columnFamilyHandle);
+ try (RocksIteratorWrapper iteratorWrapper = RocksDBOperationUtils.getRocksIterator(db, columnFamilyHandle);
RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(db)) {
iteratorWrapper.seek(beginKeyBytes);
@@ -135,7 +135,7 @@ public class RocksDBIncrementalCheckpointUtils {
/**
* check whether the bytes is before prefixBytes in the character order.
*/
- static boolean beforeThePrefixBytes(@Nonnull byte[] bytes, @Nonnull byte[] prefixBytes) {
+ public static boolean beforeThePrefixBytes(@Nonnull byte[] bytes, @Nonnull byte[] prefixBytes) {
final int prefixLength = prefixBytes.length;
for (int i = 0; i < prefixLength; ++i) {
int r = (char) prefixBytes[i] - (char) bytes[i];
@@ -155,7 +155,7 @@ public class RocksDBIncrementalCheckpointUtils {
* @return The best candidate or null if no candidate was a good fit.
*/
@Nullable
- static KeyedStateHandle chooseTheBestStateHandleForInitial(
+ public static KeyedStateHandle chooseTheBestStateHandleForInitial(
@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
@Nonnull KeyGroupRange targetKeyGroupRange) {
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index 6f67da3..cc0726c 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -29,56 +29,32 @@ import org.apache.flink.api.common.state.StateDescriptor;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
-import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.contrib.streaming.state.iterator.RocksStateKeysIterator;
import org.apache.flink.contrib.streaming.state.snapshot.RocksDBSnapshotStrategyBase;
-import org.apache.flink.contrib.streaming.state.snapshot.RocksFullSnapshotStrategy;
-import org.apache.flink.contrib.streaming.state.snapshot.RocksIncrementalSnapshotStrategy;
import org.apache.flink.contrib.streaming.state.ttl.RocksDbTtlCompactFiltersManager;
-import org.apache.flink.core.fs.FSDataInputStream;
-import org.apache.flink.core.fs.FileStatus;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.core.memory.DataInputDeserializer;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.DataOutputSerializer;
-import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.CheckpointStreamFactory;
-import org.apache.flink.runtime.state.DirectoryStateHandle;
-import org.apache.flink.runtime.state.IncrementalKeyedStateHandle;
-import org.apache.flink.runtime.state.IncrementalLocalKeyedStateHandle;
-import org.apache.flink.runtime.state.KeyExtractorFunction;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.KeyGroupedInternalPriorityQueue;
-import org.apache.flink.runtime.state.KeyGroupsStateHandle;
import org.apache.flink.runtime.state.Keyed;
-import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
import org.apache.flink.runtime.state.KeyedStateHandle;
-import org.apache.flink.runtime.state.LocalRecoveryConfig;
import org.apache.flink.runtime.state.PriorityComparable;
-import org.apache.flink.runtime.state.PriorityComparator;
import org.apache.flink.runtime.state.PriorityQueueSetFactory;
import org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo;
-import org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo;
import org.apache.flink.runtime.state.RegisteredStateMetaInfoBase;
-import org.apache.flink.runtime.state.SnappyStreamCompressionDecorator;
import org.apache.flink.runtime.state.SnapshotResult;
-import org.apache.flink.runtime.state.StateHandleID;
+import org.apache.flink.runtime.state.StateSerializerProvider;
import org.apache.flink.runtime.state.StateSnapshotTransformer.StateSnapshotTransformFactory;
import org.apache.flink.runtime.state.StreamCompressionDecorator;
-import org.apache.flink.runtime.state.StreamStateHandle;
-import org.apache.flink.runtime.state.UncompressedStreamCompressionDecorator;
import org.apache.flink.runtime.state.heap.HeapPriorityQueueElement;
import org.apache.flink.runtime.state.heap.HeapPriorityQueueSetFactory;
-import org.apache.flink.runtime.state.heap.KeyGroupPartitionedPriorityQueue;
-import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.apache.flink.util.FileUtils;
import org.apache.flink.util.FlinkRuntimeException;
@@ -87,7 +63,6 @@ import org.apache.flink.util.Preconditions;
import org.apache.flink.util.ResourceGuard;
import org.apache.flink.util.StateMigrationException;
-import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.ColumnFamilyOptions;
import org.rocksdb.DBOptions;
@@ -99,27 +74,16 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.SortedMap;
import java.util.Spliterator;
import java.util.Spliterators;
-import java.util.TreeMap;
-import java.util.UUID;
import java.util.concurrent.RunnableFuture;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -127,10 +91,6 @@ import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import static org.apache.flink.contrib.streaming.state.RocksDBSnapshotTransformFactoryAdaptor.wrapStateSnapshotTransformFactory;
-import static org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.END_OF_KEY_GROUP_MARK;
-import static org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.SST_FILE_SUFFIX;
-import static org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.clearMetaDataFollowsFlag;
-import static org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.hasMetaDataFollowsFlag;
/**
* An {@link AbstractKeyedStateBackend} that stores its state in {@code RocksDB} and serializes state to
@@ -179,9 +139,6 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
/** Path where this configured instance stores its data directory. */
private final File instanceBasePath;
- /** Path where this configured instance stores its RocksDB database. */
- private final File instanceRocksDBPath;
-
/**
* Protects access to RocksDB in other threads, like the checkpointing thread from parallel call that disposes the
* RocksDb object.
@@ -189,19 +146,6 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
private final ResourceGuard rocksDBResourceGuard;
/**
- * Our RocksDB database, this is used by the actual subclasses of {@link AbstractRocksDBState}
- * to store state. The different k/v states that we have don't each have their own RocksDB
- * instance. They all write to this instance but to their own column family.
- */
- protected RocksDB db;
-
- /**
- * We are not using the default column family for Flink state ops, but we still need to remember this handle so that
- * we can close it properly when the backend is closed. This is required by RocksDB's native memory management.
- */
- private ColumnFamilyHandle defaultColumnFamily;
-
- /**
* The write options to use in the states. We disable write ahead logging.
*/
private final WriteOptions writeOptions;
@@ -215,42 +159,45 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
/** Number of bytes required to prefix the key groups. */
private final int keyGroupPrefixBytes;
- /** True if incremental checkpointing is enabled. */
- private final boolean enableIncrementalCheckpointing;
-
/** Thread number used to transfer state files while restoring/snapshotting. */
private final int numberOfTransferingThreads;
- /** The configuration of local recovery. */
- private final LocalRecoveryConfig localRecoveryConfig;
+ /**
+ * We are not using the default column family for Flink state ops, but we still need to remember this handle so that
+ * we can close it properly when the backend is closed. Note that the one returned by {@link RocksDB#open(String)}
+ * is different from that by {@link RocksDB#getDefaultColumnFamily()}, probably it's a bug of RocksDB java API.
+ */
+ private final ColumnFamilyHandle defaultColumnFamily;
+
+ /** Shared wrapper for batch writes to the RocksDB instance. */
+ private final RocksDBWriteBatchWrapper writeBatchWrapper;
/** The checkpoint snapshot strategy, e.g., if we use full or incremental checkpoints, local state, and so on. */
- private RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy;
+ private final RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy;
/** The savepoint snapshot strategy. */
- private RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy;
+ private final RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy;
+
+ /** The native metrics monitor. */
+ private final RocksDBNativeMetricMonitor nativeMetricMonitor;
/** Factory for priority queue state. */
private final PriorityQueueSetFactory priorityQueueFactory;
- /** Shared wrapper for batch writes to the RocksDB instance. */
- private RocksDBWriteBatchWrapper writeBatchWrapper;
-
- private final RocksDBNativeMetricOptions metricOptions;
-
- private final MetricGroup metricGroup;
-
- /** The native metrics monitor. */
- private RocksDBNativeMetricMonitor nativeMetricMonitor;
-
/**
* Helper to build the byte arrays of composite keys to address data in RocksDB. Shared across all states.
- *
- * <p>We create the builder after the restore phase in the {@link #restore(Object)} method. The timing of
- * the creation is important, because only after the restore we are certain that the key serializer
- * is final after potential reconfigurations during the restore.
*/
- private RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder;
+ private final RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder;
+
+ /**
+ * Our RocksDB database, this is used by the actual subclasses of {@link AbstractRocksDBState}
+ * to store state. The different k/v states that we have don't each have their own RocksDB
+ * instance. They all write to this instance but to their own column family.
+ */
+ protected final RocksDB db;
+
+ // mark whether this backend is already disposed and prevent duplicate disposing
+ private boolean disposed = false;
private final RocksDbTtlCompactFiltersManager ttlCompactFiltersManager;
@@ -261,30 +208,35 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
DBOptions dbOptions,
Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
TaskKvStateRegistry kvStateRegistry,
- TypeSerializer<K> keySerializer,
+ StateSerializerProvider<K> keySerializerProvider,
int numberOfKeyGroups,
KeyGroupRange keyGroupRange,
ExecutionConfig executionConfig,
- boolean enableIncrementalCheckpointing,
int numberOfTransferingThreads,
- LocalRecoveryConfig localRecoveryConfig,
- RocksDBStateBackend.PriorityQueueStateType priorityQueueStateType,
TtlTimeProvider ttlTimeProvider,
- boolean enableTtlCompactionFilter,
- RocksDBNativeMetricOptions metricOptions,
- MetricGroup metricGroup
- ) throws IOException {
-
- super(kvStateRegistry, keySerializer, userCodeClassLoader,
- numberOfKeyGroups, keyGroupRange, executionConfig, ttlTimeProvider);
-
- this.ttlCompactFiltersManager = new RocksDbTtlCompactFiltersManager(enableTtlCompactionFilter);
+ RocksDB db,
+ LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
+ int keyGroupPrefixBytes,
+ CloseableRegistry cancelStreamRegistry,
+ StreamCompressionDecorator keyGroupCompressionDecorator,
+ ResourceGuard rocksDBResourceGuard,
+ RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
+ RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
+ RocksDBWriteBatchWrapper writeBatchWrapper,
+ ColumnFamilyHandle defaultColumnFamilyHandle,
+ RocksDBNativeMetricMonitor nativeMetricMonitor,
+ RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
+ PriorityQueueSetFactory priorityQueueFactory,
+ RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
+
+ super(kvStateRegistry, keySerializerProvider, userCodeClassLoader, numberOfKeyGroups,
+ keyGroupRange, executionConfig, ttlTimeProvider, cancelStreamRegistry, keyGroupCompressionDecorator);
+
+ this.ttlCompactFiltersManager = ttlCompactFiltersManager;
this.operatorIdentifier = Preconditions.checkNotNull(operatorIdentifier);
- this.enableIncrementalCheckpointing = enableIncrementalCheckpointing;
this.numberOfTransferingThreads = numberOfTransferingThreads;
- this.rocksDBResourceGuard = new ResourceGuard();
// ensure that we use the right merge operator, because other code relies on this
this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);
@@ -292,49 +244,20 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
this.dbOptions = Preconditions.checkNotNull(dbOptions);
this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);
- this.instanceRocksDBPath = new File(instanceBasePath, "db");
-
- checkAndCreateDirectory(instanceBasePath);
- if (instanceRocksDBPath.exists()) {
- // Clear the base directory when the backend is created
- // in case something crashed and the backend never reached dispose()
- cleanInstanceBasePath();
- }
-
- this.localRecoveryConfig = Preconditions.checkNotNull(localRecoveryConfig);
- this.keyGroupPrefixBytes =
- RocksDBKeySerializationUtils.computeRequiredBytesInKeyGroupPrefix(getNumberOfKeyGroups());
- this.kvStateInformation = new LinkedHashMap<>();
+ this.keyGroupPrefixBytes = keyGroupPrefixBytes;
+ this.kvStateInformation = kvStateInformation;
this.writeOptions = new WriteOptions().setDisableWAL(true);
-
- this.metricOptions = metricOptions;
- this.metricGroup = metricGroup;
-
- switch (priorityQueueStateType) {
- case HEAP:
- this.priorityQueueFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
- break;
- case ROCKSDB:
- this.priorityQueueFactory = new RocksDBPriorityQueueSetFactory();
- break;
- default:
- throw new IllegalArgumentException("Unknown priority queue state type: " + priorityQueueStateType);
- }
- }
-
- private static void checkAndCreateDirectory(File directory) throws IOException {
- if (directory.exists()) {
- if (!directory.isDirectory()) {
- throw new IOException("Not a directory: " + directory);
- }
- } else {
- if (!directory.mkdirs()) {
- throw new IOException(
- String.format("Could not create RocksDB data directory at %s.", directory));
- }
- }
+ this.db = db;
+ this.rocksDBResourceGuard = rocksDBResourceGuard;
+ this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
+ this.savepointSnapshotStrategy = savepointSnapshotStrategy;
+ this.writeBatchWrapper = writeBatchWrapper;
+ this.defaultColumnFamily = defaultColumnFamilyHandle;
+ this.nativeMetricMonitor = nativeMetricMonitor;
+ this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
+ this.priorityQueueFactory = priorityQueueFactory;
}
@SuppressWarnings("unchecked")
@@ -363,7 +286,7 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
throw new FlinkRuntimeException("Failed to get keys from RocksDB state backend.", ex);
}
- RocksIteratorWrapper iterator = getRocksIterator(db, columnInfo.columnFamilyHandle);
+ RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnInfo.columnFamilyHandle);
iterator.seekToFirst();
final RocksStateKeysIterator<K> iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes,
@@ -375,16 +298,8 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
@VisibleForTesting
ColumnFamilyHandle getColumnFamilyHandle(String state) {
- RocksDbKvStateInfo columnInfo = kvStateInformation.get(state);
- return columnInfo != null ? columnInfo.columnFamilyHandle : null;
- }
-
- private void registerKvStateInformation(String columnFamilyName, RocksDbKvStateInfo stateInfo) {
- kvStateInformation.put(columnFamilyName, stateInfo);
-
- if (nativeMetricMonitor != null) {
- nativeMetricMonitor.registerColumnFamily(columnFamilyName, stateInfo.columnFamilyHandle);
- }
+ RocksDbKvStateInfo columnInfo = kvStateInformation.get(state);
+ return columnInfo != null ? columnInfo.columnFamilyHandle : null;
}
@Override
@@ -398,6 +313,9 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
*/
@Override
public void dispose() {
+ if (this.disposed) {
+ return;
+ }
super.dispose();
// This call will block until all clients that still acquire access to the RocksDB instance have released it,
@@ -423,21 +341,18 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
// DB is closed. See:
// https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
// Start with default CF ...
- addColumnFamilyToCloseLater(columnFamilyOptions, defaultColumnFamily);
+ RocksDBOperationUtils.addColumnFamilyOptionsToCloseLater(columnFamilyOptions, defaultColumnFamily);
IOUtils.closeQuietly(defaultColumnFamily);
// ... continue with the ones created by Flink...
for (RocksDbKvStateInfo kvStateInfo : kvStateInformation.values()) {
- addColumnFamilyToCloseLater(columnFamilyOptions, kvStateInfo.columnFamilyHandle);
+ RocksDBOperationUtils.addColumnFamilyOptionsToCloseLater(columnFamilyOptions, kvStateInfo.columnFamilyHandle);
IOUtils.closeQuietly(kvStateInfo.columnFamilyHandle);
}
// ... and finally close the DB instance ...
IOUtils.closeQuietly(db);
- // invalidate the reference
- db = null;
-
columnFamilyOptions.forEach(IOUtils::closeQuietly);
IOUtils.closeQuietly(dbOptions);
@@ -449,16 +364,7 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
cleanInstanceBasePath();
}
- }
-
- private static void addColumnFamilyToCloseLater(
- List<ColumnFamilyOptions> columnFamilyOptions, ColumnFamilyHandle columnFamilyHandle) {
-
- try {
- columnFamilyOptions.add(columnFamilyHandle.getDescriptor().getOptions());
- } catch (RocksDBException e) {
- // ignore
- }
+ this.disposed = true;
}
@Nonnull
@@ -480,6 +386,10 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
}
}
+ // ------------------------------------------------------------------------
+ // Getters and Setters
+ // ------------------------------------------------------------------------
+
public int getKeyGroupPrefixBytes() {
return keyGroupPrefixBytes;
}
@@ -497,6 +407,11 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
return sharedRocksKeyBuilder;
}
+ @VisibleForTesting
+ boolean isDisposed() {
+ return this.disposed;
+ }
+
/**
* Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and
* is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always
@@ -535,96 +450,8 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
}
@Override
- public void restore(Collection<KeyedStateHandle> restoreState) throws Exception {
-
- LOG.info("Initializing RocksDB keyed state backend.");
-
- // clear all meta data
- kvStateInformation.clear();
-
- try {
- RocksDBIncrementalRestoreOperation<K> incrementalRestoreOperation = null;
- if (restoreState == null || restoreState.isEmpty()) {
- createDB();
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Restoring snapshot from state handles: {}, will use {} thread(s) to download files from DFS.", restoreState, numberOfTransferingThreads);
- }
-
- KeyedStateHandle firstStateHandle = restoreState.iterator().next();
- if (firstStateHandle instanceof IncrementalKeyedStateHandle
- || firstStateHandle instanceof IncrementalLocalKeyedStateHandle) {
- incrementalRestoreOperation = new RocksDBIncrementalRestoreOperation<>(this);
- incrementalRestoreOperation.restore(restoreState);
- } else {
- RocksDBFullRestoreOperation<K> fullRestoreOperation = new RocksDBFullRestoreOperation<>(this);
- fullRestoreOperation.doRestore(restoreState);
- }
- }
-
- // it is important that we only create the key builder after the restore, and not before;
- // restore operations may reconfigure the key serializer, so accessing the key serializer
- // only now we can be certain that the key serializer used in the builder is final.
- this.sharedRocksKeyBuilder = new RocksDBSerializedCompositeKeyBuilder<>(
- getKeySerializer(),
- keyGroupPrefixBytes,
- 32);
-
- initializeSnapshotStrategy(incrementalRestoreOperation);
- } catch (Exception ex) {
- dispose();
- throw ex;
- }
- }
-
- @VisibleForTesting
- void initializeSnapshotStrategy(@Nullable RocksDBIncrementalRestoreOperation<K> incrementalRestoreOperation) {
-
- this.savepointSnapshotStrategy =
- new RocksFullSnapshotStrategy<>(
- db,
- rocksDBResourceGuard,
- getKeySerializer(),
- kvStateInformation,
- keyGroupRange,
- keyGroupPrefixBytes,
- localRecoveryConfig,
- cancelStreamRegistry,
- keyGroupCompressionDecorator);
-
- if (enableIncrementalCheckpointing) {
- final UUID backendUID;
- final SortedMap<Long, Set<StateHandleID>> materializedSstFiles;
- final long lastCompletedCheckpointId;
-
- if (incrementalRestoreOperation == null) {
- backendUID = UUID.randomUUID();
- materializedSstFiles = new TreeMap<>();
- lastCompletedCheckpointId = -1L;
- } else {
- backendUID = Preconditions.checkNotNull(incrementalRestoreOperation.getRestoredBackendUID());
- materializedSstFiles = Preconditions.checkNotNull(incrementalRestoreOperation.getRestoredSstFiles());
- lastCompletedCheckpointId = incrementalRestoreOperation.getLastCompletedCheckpointId();
- Preconditions.checkState(lastCompletedCheckpointId >= 0L);
- }
- // TODO eventually we might want to separate savepoint and snapshot strategy, i.e. having 2 strategies.
- this.checkpointSnapshotStrategy = new RocksIncrementalSnapshotStrategy<>(
- db,
- rocksDBResourceGuard,
- getKeySerializer(),
- kvStateInformation,
- keyGroupRange,
- keyGroupPrefixBytes,
- localRecoveryConfig,
- cancelStreamRegistry,
- instanceBasePath,
- backendUID,
- materializedSstFiles,
- lastCompletedCheckpointId,
- numberOfTransferingThreads);
- } else {
- this.checkpointSnapshotStrategy = savepointSnapshotStrategy;
- }
+ public void restore(Collection<KeyedStateHandle> restoreState) {
+ // all restore work done in builder and nothing to do here
}
@Override
@@ -639,728 +466,6 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
}
}
- private void createDB() throws IOException {
- List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(1);
- this.db = openDB(instanceRocksDBPath.getAbsolutePath(), Collections.emptyList(), columnFamilyHandles);
- this.writeBatchWrapper = new RocksDBWriteBatchWrapper(db, writeOptions);
- this.defaultColumnFamily = columnFamilyHandles.get(0);
- }
-
- private RocksDB openDB(
- String path,
- List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
- List<ColumnFamilyHandle> stateColumnFamilyHandles) throws IOException {
-
- List<ColumnFamilyDescriptor> columnFamilyDescriptors =
- new ArrayList<>(1 + stateColumnFamilyDescriptors.size());
-
- // we add the required descriptor for the default CF in FIRST position, see
- // https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
- columnFamilyDescriptors.add(new ColumnFamilyDescriptor(
- RocksDB.DEFAULT_COLUMN_FAMILY,
- createColumnFamilyOptions("default")));
- columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);
-
- RocksDB dbRef;
-
- try {
- dbRef = RocksDB.open(
- Preconditions.checkNotNull(dbOptions),
- Preconditions.checkNotNull(path),
- columnFamilyDescriptors,
- stateColumnFamilyHandles);
- } catch (RocksDBException e) {
- throw new IOException("Error while opening RocksDB instance.", e);
- }
-
- // requested + default CF
- Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(),
- "Not all requested column family handles have been created");
-
- if (this.metricOptions.isEnabled()) {
- this.nativeMetricMonitor = new RocksDBNativeMetricMonitor(
- dbRef,
- metricOptions,
- metricGroup
- );
- }
-
- return dbRef;
- }
-
- /**
- * Encapsulates the process of restoring a RocksDBKeyedStateBackend from a full snapshot.
- */
- private static final class RocksDBFullRestoreOperation<K> {
-
- private final RocksDBKeyedStateBackend<K> rocksDBKeyedStateBackend;
-
- /** Current key-groups state handle from which we restore key-groups. */
- private KeyGroupsStateHandle currentKeyGroupsStateHandle;
- /** Current input stream we obtained from currentKeyGroupsStateHandle. */
- private FSDataInputStream currentStateHandleInStream;
- /** Current data input view that wraps currentStateHandleInStream. */
- private DataInputView currentStateHandleInView;
- /** Current list of ColumnFamilyHandles for all column families we restore from currentKeyGroupsStateHandle. */
- private List<RocksDbKvStateInfo> currentKvStates;
- /** The compression decorator that was used for writing the state, as determined by the meta data. */
- private StreamCompressionDecorator keygroupStreamCompressionDecorator;
-
- private boolean isKeySerializerCompatibilityChecked;
-
- /**
- * Creates a restore operation object for the given state backend instance.
- *
- * @param rocksDBKeyedStateBackend the state backend into which we restore
- */
- RocksDBFullRestoreOperation(RocksDBKeyedStateBackend<K> rocksDBKeyedStateBackend) {
- this.rocksDBKeyedStateBackend = Preconditions.checkNotNull(rocksDBKeyedStateBackend);
- }
-
- /**
- * Restores all key-groups data that is referenced by the passed state handles.
- *
- * @param keyedStateHandles List of all key groups state handles that shall be restored.
- */
- void doRestore(Collection<KeyedStateHandle> keyedStateHandles)
- throws IOException, StateMigrationException, RocksDBException {
-
- rocksDBKeyedStateBackend.createDB();
-
- for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {
- if (keyedStateHandle != null) {
-
- if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
- throw new IllegalStateException("Unexpected state handle type, " +
- "expected: " + KeyGroupsStateHandle.class +
- ", but found: " + keyedStateHandle.getClass());
- }
- this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
- restoreKeyGroupsInStateHandle();
- }
- }
- }
-
- /**
- * Restore one key groups state handle.
- */
- private void restoreKeyGroupsInStateHandle()
- throws IOException, StateMigrationException, RocksDBException {
- try {
- currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream();
- rocksDBKeyedStateBackend.cancelStreamRegistry.registerCloseable(currentStateHandleInStream);
- currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream);
- restoreKVStateMetaData();
- restoreKVStateData();
- } finally {
- if (rocksDBKeyedStateBackend.cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) {
- IOUtils.closeQuietly(currentStateHandleInStream);
- }
- }
- }
-
- /**
- * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
- */
- private void restoreKVStateMetaData() throws IOException, StateMigrationException {
-
- // isSerializerPresenceRequired flag is set to false, since for the RocksDB state backend,
- // deserialization of state happens lazily during runtime; we depend on the fact
- // that the new serializer for states could be compatible, and therefore the restore can continue
- // without old serializers required to be present.
- KeyedBackendSerializationProxy<K> serializationProxy =
- new KeyedBackendSerializationProxy<>(rocksDBKeyedStateBackend.userCodeClassLoader);
-
- serializationProxy.read(currentStateHandleInView);
-
- if (!isKeySerializerCompatibilityChecked) {
- // check for key serializer compatibility; this also reconfigures the
- // key serializer to be compatible, if it is required and is possible
- TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
- rocksDBKeyedStateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerSnapshot());
- if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
- throw new StateMigrationException("The new key serializer must be compatible.");
- }
-
- isKeySerializerCompatibilityChecked = true;
- }
-
- this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
- SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;
-
- List<StateMetaInfoSnapshot> restoredMetaInfos =
- serializationProxy.getStateMetaInfoSnapshots();
- currentKvStates = new ArrayList<>(restoredMetaInfos.size());
-
- for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
-
- RocksDbKvStateInfo registeredColumn =
- rocksDBKeyedStateBackend.kvStateInformation.get(restoredMetaInfo.getName());
-
- if (registeredColumn == null) {
- // create a meta info for the state on restore;
- // this allows us to retain the state in future snapshots even if it wasn't accessed
- RegisteredStateMetaInfoBase metaInfoBase =
- RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(restoredMetaInfo);
- registeredColumn = rocksDBKeyedStateBackend.createStateInfo(metaInfoBase);
- rocksDBKeyedStateBackend.kvStateInformation.put(restoredMetaInfo.getName(), registeredColumn);
-
- } else {
- // TODO with eager state registration in place, check here for serializer migration strategies
- }
- currentKvStates.add(registeredColumn);
- }
- }
-
- /**
- * Restore the KV-state / ColumnFamily data for all key-groups referenced by the current state handle.
- */
- private void restoreKVStateData() throws IOException, RocksDBException {
- //for all key-groups in the current state handle...
- try (RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(rocksDBKeyedStateBackend.db)) {
- for (Tuple2<Integer, Long> keyGroupOffset : currentKeyGroupsStateHandle.getGroupRangeOffsets()) {
- int keyGroup = keyGroupOffset.f0;
-
- // Check that restored key groups all belong to the backend
- Preconditions.checkState(rocksDBKeyedStateBackend.getKeyGroupRange().contains(keyGroup),
- "The key group must belong to the backend");
-
- long offset = keyGroupOffset.f1;
- //not empty key-group?
- if (0L != offset) {
- currentStateHandleInStream.seek(offset);
- try (InputStream compressedKgIn = keygroupStreamCompressionDecorator.decorateWithCompression(currentStateHandleInStream)) {
- DataInputViewStreamWrapper compressedKgInputView = new DataInputViewStreamWrapper(compressedKgIn);
- //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
- int kvStateId = compressedKgInputView.readShort();
- RocksDbKvStateInfo stateInfo = currentKvStates.get(kvStateId);
- //insert all k/v pairs into DB
- boolean keyGroupHasMoreKeys = true;
- while (keyGroupHasMoreKeys) {
- byte[] key = BytePrimitiveArraySerializer.INSTANCE.deserialize(compressedKgInputView);
- byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(compressedKgInputView);
- if (hasMetaDataFollowsFlag(key)) {
- //clear the signal bit in the key to make it ready for insertion again
- clearMetaDataFollowsFlag(key);
- writeBatchWrapper.put(stateInfo.columnFamilyHandle, key, value);
- //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
- kvStateId = END_OF_KEY_GROUP_MARK
- & compressedKgInputView.readShort();
- if (END_OF_KEY_GROUP_MARK == kvStateId) {
- keyGroupHasMoreKeys = false;
- } else {
- stateInfo = currentKvStates.get(kvStateId);
- }
- } else {
- writeBatchWrapper.put(stateInfo.columnFamilyHandle, key, value);
- }
- }
- }
- }
- }
- }
- }
- }
-
- /**
- * Encapsulates the process of restoring a RocksDBKeyedStateBackend from an incremental snapshot.
- */
- private static class RocksDBIncrementalRestoreOperation<T> {
-
- private final RocksDBKeyedStateBackend<T> stateBackend;
- private final SortedMap<Long, Set<StateHandleID>> restoredSstFiles;
- private UUID restoredBackendUID;
- private long lastCompletedCheckpointId;
- private boolean isKeySerializerCompatibilityChecked;
-
- private RocksDBIncrementalRestoreOperation(RocksDBKeyedStateBackend<T> stateBackend) {
-
- this.stateBackend = stateBackend;
- this.restoredSstFiles = new TreeMap<>();
- }
-
- SortedMap<Long, Set<StateHandleID>> getRestoredSstFiles() {
- return restoredSstFiles;
- }
-
- UUID getRestoredBackendUID() {
- return restoredBackendUID;
- }
-
- long getLastCompletedCheckpointId() {
- return lastCompletedCheckpointId;
- }
-
- /**
- * Root method that branches for different implementations of {@link KeyedStateHandle}.
- */
- void restore(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {
-
- if (restoreStateHandles.isEmpty()) {
- return;
- }
-
- final KeyedStateHandle theFirstStateHandle = restoreStateHandles.iterator().next();
-
- boolean isRescaling = (restoreStateHandles.size() > 1 ||
- !Objects.equals(theFirstStateHandle.getKeyGroupRange(), stateBackend.keyGroupRange));
-
- if (!isRescaling) {
- restoreWithoutRescaling(theFirstStateHandle);
- } else {
- restoreWithRescaling(restoreStateHandles);
- }
- }
-
- /**
- * Recovery from a single remote incremental state without rescaling.
- */
- void restoreWithoutRescaling(KeyedStateHandle rawStateHandle) throws Exception {
-
- IncrementalLocalKeyedStateHandle localKeyedStateHandle;
- List<StateMetaInfoSnapshot> stateMetaInfoSnapshots;
- List<ColumnFamilyDescriptor> descriptors;
-
- // Recovery from remote incremental state.
- Path temporaryRestoreInstancePath = new Path(
- stateBackend.instanceBasePath.getAbsolutePath(),
- UUID.randomUUID().toString());
-
- try {
- if (rawStateHandle instanceof IncrementalKeyedStateHandle) {
-
- IncrementalKeyedStateHandle restoreStateHandle = (IncrementalKeyedStateHandle) rawStateHandle;
-
- // read state data.
- try (RocksDBStateDownloader rocksDBStateDownloader =
- new RocksDBStateDownloader(stateBackend.numberOfTransferingThreads)) {
- rocksDBStateDownloader.transferAllStateDataToDirectory(
- restoreStateHandle,
- temporaryRestoreInstancePath,
- stateBackend.cancelStreamRegistry);
- }
-
- stateMetaInfoSnapshots = readMetaData(restoreStateHandle.getMetaStateHandle());
- descriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots);
-
- // since we transferred all remote state to a local directory, we can use the same code as for
- // local recovery.
- localKeyedStateHandle = new IncrementalLocalKeyedStateHandle(
- restoreStateHandle.getBackendIdentifier(),
- restoreStateHandle.getCheckpointId(),
- new DirectoryStateHandle(temporaryRestoreInstancePath),
- restoreStateHandle.getKeyGroupRange(),
- restoreStateHandle.getMetaStateHandle(),
- restoreStateHandle.getSharedState().keySet());
- } else if (rawStateHandle instanceof IncrementalLocalKeyedStateHandle) {
-
- // Recovery from local incremental state.
- localKeyedStateHandle = (IncrementalLocalKeyedStateHandle) rawStateHandle;
- stateMetaInfoSnapshots = readMetaData(localKeyedStateHandle.getMetaDataState());
- descriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots);
- } else {
- throw new IllegalStateException("Unexpected state handle type, " +
- "expected " + IncrementalKeyedStateHandle.class + " or " + IncrementalLocalKeyedStateHandle.class +
- ", but found " + rawStateHandle.getClass());
- }
-
- restoreLocalStateIntoFullInstance(
- localKeyedStateHandle,
- descriptors,
- stateMetaInfoSnapshots);
- } finally {
- FileSystem restoreFileSystem = temporaryRestoreInstancePath.getFileSystem();
- if (restoreFileSystem.exists(temporaryRestoreInstancePath)) {
- restoreFileSystem.delete(temporaryRestoreInstancePath, true);
- }
- }
- }
-
- /**
- * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary
- * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the
- * real restore instance and then the temporary instance is discarded.
- */
- void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {
-
- this.restoredBackendUID = UUID.randomUUID();
-
- initTargetDB(restoreStateHandles, stateBackend.keyGroupRange);
-
- byte[] startKeyGroupPrefixBytes = new byte[stateBackend.keyGroupPrefixBytes];
- RocksDBKeySerializationUtils.serializeKeyGroup(stateBackend.getKeyGroupRange().getStartKeyGroup(), startKeyGroupPrefixBytes);
-
- byte[] stopKeyGroupPrefixBytes = new byte[stateBackend.keyGroupPrefixBytes];
- RocksDBKeySerializationUtils.serializeKeyGroup(stateBackend.getKeyGroupRange().getEndKeyGroup() + 1, stopKeyGroupPrefixBytes);
-
- for (KeyedStateHandle rawStateHandle : restoreStateHandles) {
-
- if (!(rawStateHandle instanceof IncrementalKeyedStateHandle)) {
- throw new IllegalStateException("Unexpected state handle type, " +
- "expected " + IncrementalKeyedStateHandle.class +
- ", but found " + rawStateHandle.getClass());
- }
-
- Path temporaryRestoreInstancePath = new Path(stateBackend.instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString());
- try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle(
- (IncrementalKeyedStateHandle) rawStateHandle, temporaryRestoreInstancePath, false);
- RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(stateBackend.db)) {
-
- List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors;
- List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles;
-
- // iterating only the requested descriptors automatically skips the default column family handle
- for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) {
- ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i);
- ColumnFamilyDescriptor tmpColumnFamilyDescriptor = tmpColumnFamilyDescriptors.get(i);
-
- ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterColumnFamilyHandle(
- tmpColumnFamilyDescriptor, null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i));
-
- try (RocksIteratorWrapper iterator = getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) {
-
- iterator.seek(startKeyGroupPrefixBytes);
-
- while (iterator.isValid()) {
-
- if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) {
- writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value());
- } else {
- // Since the iterator will visit the record according to the sorted order,
- // we can just break here.
- break;
- }
-
- iterator.next();
- }
- } // releases native iterator resources
- }
- } finally {
- FileSystem restoreFileSystem = temporaryRestoreInstancePath.getFileSystem();
- if (restoreFileSystem.exists(temporaryRestoreInstancePath)) {
- restoreFileSystem.delete(temporaryRestoreInstancePath, true);
- }
- }
- }
- }
-
- private class RestoredDBInstance implements AutoCloseable {
-
- @Nonnull
- private final RocksDB db;
-
- @Nonnull
- private final ColumnFamilyHandle defaultColumnFamilyHandle;
-
- @Nonnull
- private final List<ColumnFamilyHandle> columnFamilyHandles;
-
- @Nonnull
- private final List<ColumnFamilyDescriptor> columnFamilyDescriptors;
-
- @Nonnull
- private final List<StateMetaInfoSnapshot> stateMetaInfoSnapshots;
-
- private RestoredDBInstance(
- @Nonnull RocksDB db,
- @Nonnull List<ColumnFamilyHandle> columnFamilyHandles,
- @Nonnull List<ColumnFamilyDescriptor> columnFamilyDescriptors,
- @Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {
- this.db = db;
- this.columnFamilyHandles = columnFamilyHandles;
- this.defaultColumnFamilyHandle = this.columnFamilyHandles.remove(0);
- this.columnFamilyDescriptors = columnFamilyDescriptors;
- this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
- }
-
- @Override
- public void close() {
-
- IOUtils.closeQuietly(defaultColumnFamilyHandle);
-
- for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) {
- IOUtils.closeQuietly(columnFamilyHandle);
- }
-
- IOUtils.closeQuietly(db);
- }
- }
-
- private RestoredDBInstance restoreDBInstanceFromStateHandle(
- IncrementalKeyedStateHandle restoreStateHandle,
- Path temporaryRestoreInstancePath,
- boolean registerTtlCompactFilter) throws Exception {
-
- try (RocksDBStateDownloader rocksDBStateDownloader =
- new RocksDBStateDownloader(stateBackend.numberOfTransferingThreads)) {
- rocksDBStateDownloader.transferAllStateDataToDirectory(
- restoreStateHandle,
- temporaryRestoreInstancePath,
- stateBackend.cancelStreamRegistry);
- }
-
- // read meta data
- List<StateMetaInfoSnapshot> stateMetaInfoSnapshots =
- readMetaData(restoreStateHandle.getMetaStateHandle());
-
- List<ColumnFamilyDescriptor> descriptors =
- createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, registerTtlCompactFilter);
-
- List<ColumnFamilyHandle> columnFamilyHandles =
- new ArrayList<>(stateMetaInfoSnapshots.size() + 1);
-
- RocksDB restoreDb = stateBackend.openDB(
- temporaryRestoreInstancePath.getPath(),
- descriptors,
- columnFamilyHandles);
-
- return new RestoredDBInstance(restoreDb, columnFamilyHandles, descriptors, stateMetaInfoSnapshots);
- }
-
- private ColumnFamilyHandle getOrRegisterColumnFamilyHandle(
- ColumnFamilyDescriptor columnFamilyDescriptor,
- ColumnFamilyHandle columnFamilyHandle,
- StateMetaInfoSnapshot stateMetaInfoSnapshot) throws RocksDBException {
-
- RocksDbKvStateInfo registeredStateMetaInfoEntry =
- stateBackend.kvStateInformation.get(stateMetaInfoSnapshot.getName());
-
- if (null == registeredStateMetaInfoEntry) {
- // create a meta info for the state on restore;
- // this allows us to retain the state in future snapshots even if it wasn't accessed
- RegisteredStateMetaInfoBase stateMetaInfo =
- RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot);
- registeredStateMetaInfoEntry =
- new RocksDbKvStateInfo(
- columnFamilyHandle != null ? columnFamilyHandle : stateBackend.db.createColumnFamily(columnFamilyDescriptor),
- stateMetaInfo);
-
- stateBackend.registerKvStateInformation(
- stateMetaInfoSnapshot.getName(),
- registeredStateMetaInfoEntry);
- }
-
- return registeredStateMetaInfoEntry.columnFamilyHandle;
- }
-
- /**
- * This method first try to find a initial handle to init the target db, if the initial handle
- * is not null, we just init the target db with the handle and clip it with the target key-group
- * range. If the initial handle is null we create a empty db as the target db.
- */
- private void initTargetDB(
- Collection<KeyedStateHandle> restoreStateHandles,
- KeyGroupRange targetKeyGroupRange) throws Exception {
-
- IncrementalKeyedStateHandle initialHandle = (IncrementalKeyedStateHandle) RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial(
- restoreStateHandles, targetKeyGroupRange);
-
- if (initialHandle != null) {
- restoreStateHandles.remove(initialHandle);
- RestoredDBInstance restoreDBInfo = null;
- Path instancePath = new Path(stateBackend.instanceRocksDBPath.getAbsolutePath());
- try {
- restoreDBInfo = restoreDBInstanceFromStateHandle(
- initialHandle, instancePath, true);
-
- RocksDBIncrementalCheckpointUtils.clipDBWithKeyGroupRange(
- restoreDBInfo.db,
- restoreDBInfo.columnFamilyHandles,
- targetKeyGroupRange,
- initialHandle.getKeyGroupRange(),
- stateBackend.keyGroupPrefixBytes);
-
- stateBackend.db = restoreDBInfo.db;
- stateBackend.defaultColumnFamily = restoreDBInfo.defaultColumnFamilyHandle;
- stateBackend.writeBatchWrapper =
- new RocksDBWriteBatchWrapper(stateBackend.db, stateBackend.writeOptions);
-
- for (int i = 0; i < restoreDBInfo.stateMetaInfoSnapshots.size(); ++i) {
- getOrRegisterColumnFamilyHandle(
- restoreDBInfo.columnFamilyDescriptors.get(i),
- restoreDBInfo.columnFamilyHandles.get(i),
- restoreDBInfo.stateMetaInfoSnapshots.get(i));
- }
- } catch (Exception e) {
- if (restoreDBInfo != null) {
- restoreDBInfo.close();
- }
- FileSystem restoreFileSystem = instancePath.getFileSystem();
- if (restoreFileSystem.exists(instancePath)) {
- restoreFileSystem.delete(instancePath, true);
- }
- throw e;
- }
- } else {
- List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(1);
- stateBackend.db = stateBackend.openDB(
- stateBackend.instanceRocksDBPath.getAbsolutePath(),
- Collections.emptyList(),
- columnFamilyHandles);
- stateBackend.defaultColumnFamily = columnFamilyHandles.get(0);
- stateBackend.writeBatchWrapper =
- new RocksDBWriteBatchWrapper(stateBackend.db, stateBackend.writeOptions);
- }
- }
-
- /**
- * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot.
- */
- private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors(
- List<StateMetaInfoSnapshot> stateMetaInfoSnapshots, boolean registerTtlCompactFilter) {
-
- List<ColumnFamilyDescriptor> columnFamilyDescriptors =
- new ArrayList<>(stateMetaInfoSnapshots.size());
-
- for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) {
- ColumnFamilyOptions options = stateBackend.createColumnFamilyOptions(stateMetaInfoSnapshot.getName());
- if (registerTtlCompactFilter) {
- stateBackend.ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(
- stateBackend.ttlTimeProvider, stateMetaInfoSnapshot, options);
- }
- ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(
- stateMetaInfoSnapshot.getName().getBytes(ConfigConstants.DEFAULT_CHARSET),
- options);
-
- columnFamilyDescriptors.add(columnFamilyDescriptor);
- }
- return columnFamilyDescriptors;
- }
-
- private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors(
- List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {
- return createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, true);
- }
-
- /**
- * This method implements the core of the restore logic that unifies how local and remote state are recovered.
- */
- private void restoreLocalStateIntoFullInstance(
- IncrementalLocalKeyedStateHandle restoreStateHandle,
- List<ColumnFamilyDescriptor> columnFamilyDescriptors,
- List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) throws Exception {
- // pick up again the old backend id, so the we can reference existing state
- this.restoredBackendUID = restoreStateHandle.getBackendIdentifier();
-
- LOG.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.",
- stateBackend.operatorIdentifier, this.restoredBackendUID);
-
- // create hard links in the instance directory
- if (!stateBackend.instanceRocksDBPath.mkdirs()) {
- throw new IOException("Could not create RocksDB data directory.");
- }
-
- Path restoreSourcePath = restoreStateHandle.getDirectoryStateHandle().getDirectory();
- restoreInstanceDirectoryFromPath(restoreSourcePath);
-
- List<ColumnFamilyHandle> columnFamilyHandles =
- new ArrayList<>(1 + columnFamilyDescriptors.size());
-
- stateBackend.db = stateBackend.openDB(
- stateBackend.instanceRocksDBPath.getAbsolutePath(),
- columnFamilyDescriptors, columnFamilyHandles);
-
- // extract and store the default column family which is located at the first index
- stateBackend.defaultColumnFamily = columnFamilyHandles.remove(0);
- stateBackend.writeBatchWrapper = new RocksDBWriteBatchWrapper(stateBackend.db, stateBackend.writeOptions);
-
- for (int i = 0; i < columnFamilyDescriptors.size(); ++i) {
- StateMetaInfoSnapshot stateMetaInfoSnapshot = stateMetaInfoSnapshots.get(i);
-
- ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i);
-
- // create a meta info for the state on restore;
- // this allows us to retain the state in future snapshots even if it wasn't accessed
- RegisteredStateMetaInfoBase stateMetaInfo =
- RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot);
- stateBackend.registerKvStateInformation(
- stateMetaInfoSnapshot.getName(),
- new RocksDbKvStateInfo(columnFamilyHandle, stateMetaInfo));
- }
-
- // use the restore sst files as the base for succeeding checkpoints
- restoredSstFiles.put(
- restoreStateHandle.getCheckpointId(),
- restoreStateHandle.getSharedStateHandleIDs());
-
- lastCompletedCheckpointId = restoreStateHandle.getCheckpointId();
- }
-
- /**
- * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
- * a local state.
- */
- private void restoreInstanceDirectoryFromPath(Path source) throws IOException {
-
- FileSystem fileSystem = source.getFileSystem();
-
- final FileStatus[] fileStatuses = fileSystem.listStatus(source);
-
- if (fileStatuses == null) {
- throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
- }
-
- for (FileStatus fileStatus : fileStatuses) {
- final Path filePath = fileStatus.getPath();
- final String fileName = filePath.getName();
- File restoreFile = new File(source.getPath(), fileName);
- File targetFile = new File(stateBackend.instanceRocksDBPath.getPath(), fileName);
- if (fileName.endsWith(SST_FILE_SUFFIX)) {
- // hardlink'ing the immutable sst-files.
- Files.createLink(targetFile.toPath(), restoreFile.toPath());
- } else {
- // true copy for all other files.
- Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
- }
- }
- }
-
- /**
- * Reads Flink's state meta data file from the state handle.
- */
- private List<StateMetaInfoSnapshot> readMetaData(
- StreamStateHandle metaStateHandle) throws Exception {
-
- FSDataInputStream inputStream = null;
-
- try {
- inputStream = metaStateHandle.openInputStream();
- stateBackend.cancelStreamRegistry.registerCloseable(inputStream);
-
- // isSerializerPresenceRequired flag is set to false, since for the RocksDB state backend,
- // deserialization of state happens lazily during runtime; we depend on the fact
- // that the new serializer for states could be compatible, and therefore the restore can continue
- // without old serializers required to be present.
- KeyedBackendSerializationProxy<T> serializationProxy =
- new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader);
- DataInputView in = new DataInputViewStreamWrapper(inputStream);
- serializationProxy.read(in);
-
- if (!isKeySerializerCompatibilityChecked) {
- // check for key serializer compatibility; this also reconfigures the
- // key serializer to be compatible, if it is required and is possible
- TypeSerializerSchemaCompatibility<T> keySerializerSchemaCompat =
- stateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerSnapshot());
- if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
- throw new StateMigrationException("The new key serializer must be compatible.");
- }
-
- isKeySerializerCompatibilityChecked = true;
- }
-
- return serializationProxy.getStateMetaInfoSnapshots();
- } finally {
- if (stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) {
- inputStream.close();
- }
- }
- }
- }
-
- // ------------------------------------------------------------------------
- // State factories
- // ------------------------------------------------------------------------
-
/**
* Registers a k/v state information, which includes its state id, type, RocksDB column family handle, and serializers.
*
@@ -1382,7 +487,8 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
RegisteredKeyValueStateBackendMetaInfo<N, SV> newMetaInfo;
if (oldStateInfo != null) {
@SuppressWarnings("unchecked")
- RegisteredKeyValueStateBackendMetaInfo<N, SV> castedMetaInfo = (RegisteredKeyValueStateBackendMetaInfo<N, SV>) oldStateInfo.metaInfo;
+ RegisteredKeyValueStateBackendMetaInfo<N, SV> castedMetaInfo =
+ (RegisteredKeyValueStateBackendMetaInfo<N, SV>) oldStateInfo.metaInfo;
newMetaInfo = updateRestoredStateMetaInfo(
Tuple2.of(oldStateInfo.columnFamilyHandle, castedMetaInfo),
@@ -1392,7 +498,6 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
newRocksStateInfo = new RocksDbKvStateInfo(oldStateInfo.columnFamilyHandle, newMetaInfo);
kvStateInformation.put(stateDesc.getName(), newRocksStateInfo);
-
} else {
newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
stateDesc.getType(),
@@ -1401,8 +506,10 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
stateSerializer,
StateSnapshotTransformFactory.noTransform());
- newRocksStateInfo = createStateInfo(newMetaInfo);
- registerKvStateInformation(stateDesc.getName(), newRocksStateInfo);
+ newRocksStateInfo = RocksDBOperationUtils.createStateInfo(newMetaInfo, ttlCompactFiltersManager,
+ ttlTimeProvider, db, columnFamilyOptionsFactory);
+ RocksDBOperationUtils.registerKvStateInformation(this.kvStateInformation, this.nativeMetricMonitor,
+ stateDesc.getName(), newRocksStateInfo);
}
StateSnapshotTransformFactory<SV> wrappedSnapshotTransformFactory = wrapStateSnapshotTransformFactory(
@@ -1482,7 +589,7 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
Snapshot rocksDBSnapshot = db.getSnapshot();
try (
- RocksIteratorWrapper iterator = getRocksIterator(db, stateMetaInfo.f0);
+ RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, stateMetaInfo.f0);
RocksDBWriteBatchWrapper batchWriter = new RocksDBWriteBatchWrapper(db, getWriteOptions())
) {
iterator.seekToFirst();
@@ -1509,37 +616,6 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
}
}
- /**
- * Creates a state info from a new meta info to use with a k/v state.
- */
- private RocksDbKvStateInfo createStateInfo(RegisteredStateMetaInfoBase metaInfoBase) {
- ColumnFamilyOptions options = createColumnFamilyOptions(metaInfoBase.getName());
- ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(ttlTimeProvider, metaInfoBase, options);
- String name = metaInfoBase.getName();
- return new RocksDbKvStateInfo(createColumnFamily(options, name), metaInfoBase);
- }
-
- /**
- * Creates a column family handle for use with a k/v state.
- */
- private ColumnFamilyHandle createColumnFamily(ColumnFamilyOptions options, String stateName) {
- byte[] nameBytes = stateName.getBytes(ConfigConstants.DEFAULT_CHARSET);
- Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes),
- "The chosen state name 'default' collides with the name of the default column family!");
-
- ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(nameBytes, options);
-
- try {
- return db.createColumnFamily(columnDescriptor);
- } catch (RocksDBException e) {
- throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e);
- }
- }
-
- private ColumnFamilyOptions createColumnFamilyOptions(String stateName) {
- return columnFamilyOptionsFactory.apply(stateName).setMergeOperatorName(MERGE_OPERATOR_NAME);
- }
-
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
@@ -1577,7 +653,7 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
for (RocksDbKvStateInfo metaInfo : kvStateInformation.values()) {
//TODO maybe filterOrTransform only for k/v states
- try (RocksIteratorWrapper rocksIterator = getRocksIterator(db, metaInfo.columnFamilyHandle)) {
+ try (RocksIteratorWrapper rocksIterator = RocksDBOperationUtils.getRocksIterator(db, metaInfo.columnFamilyHandle)) {
rocksIterator.seekToFirst();
while (rocksIterator.isValid()) {
@@ -1590,140 +666,27 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
return count;
}
- public static RocksIteratorWrapper getRocksIterator(RocksDB db) {
- return new RocksIteratorWrapper(db.newIterator());
- }
-
- static RocksIteratorWrapper getRocksIterator(
- RocksDB db,
- ColumnFamilyHandle columnFamilyHandle) {
- return new RocksIteratorWrapper(db.newIterator(columnFamilyHandle));
- }
-
- /**
- * Encapsulates the logic and resources in connection with creating priority queue state structures.
- */
- class RocksDBPriorityQueueSetFactory implements PriorityQueueSetFactory {
-
- /** Default cache size per key-group. */
- private static final int DEFAULT_CACHES_SIZE = 128; //TODO make this configurable
-
- /** A shared buffer to serialize elements for the priority queue. */
- @Nonnull
- private final DataOutputSerializer sharedElementOutView;
-
- /** A shared buffer to de-serialize elements for the priority queue. */
- @Nonnull
- private final DataInputDeserializer sharedElementInView;
-
- RocksDBPriorityQueueSetFactory() {
- this.sharedElementOutView = new DataOutputSerializer(128);
- this.sharedElementInView = new DataInputDeserializer();
- }
-
- @Nonnull
- @Override
- public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T>
- create(@Nonnull String stateName, @Nonnull TypeSerializer<T> byteOrderedElementSerializer) {
-
- final RocksDbKvStateInfo stateInfo =
- tryRegisterPriorityQueueMetaInfo(stateName, byteOrderedElementSerializer);
-
- return new KeyGroupPartitionedPriorityQueue<>(
- KeyExtractorFunction.forKeyedObjects(),
- PriorityComparator.forPriorityComparableObjects(),
- new KeyGroupPartitionedPriorityQueue.PartitionQueueSetFactory<T, RocksDBCachingPriorityQueueSet<T>>() {
- @Nonnull
- @Override
- public RocksDBCachingPriorityQueueSet<T> create(
- int keyGroupId,
- int numKeyGroups,
- @Nonnull KeyExtractorFunction<T> keyExtractor,
- @Nonnull PriorityComparator<T> elementPriorityComparator) {
- TreeOrderedSetCache orderedSetCache = new TreeOrderedSetCache(DEFAULT_CACHES_SIZE);
- return new RocksDBCachingPriorityQueueSet<>(
- keyGroupId,
- keyGroupPrefixBytes,
- db,
- stateInfo.columnFamilyHandle,
- byteOrderedElementSerializer,
- sharedElementOutView,
- sharedElementInView,
- writeBatchWrapper,
- orderedSetCache
- );
- }
- },
- keyGroupRange,
- numberOfKeyGroups);
- }
- }
-
- @Nonnull
- private <T> RocksDbKvStateInfo tryRegisterPriorityQueueMetaInfo(
- @Nonnull String stateName,
- @Nonnull TypeSerializer<T> byteOrderedElementSerializer) {
-
- RocksDbKvStateInfo stateInfo = kvStateInformation.get(stateName);
-
- if (stateInfo == null) {
- final ColumnFamilyHandle columnFamilyHandle = createColumnFamily(createColumnFamilyOptions(stateName), stateName);
-
- RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo =
- new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
-
- stateInfo = new RocksDbKvStateInfo(columnFamilyHandle, metaInfo);
- registerKvStateInformation(stateName, stateInfo);
- } else {
- // TODO we implement the simple way of supporting the current functionality, mimicking keyed state
- // because this should be reworked in FLINK-9376 and then we should have a common algorithm over
- // StateMetaInfoSnapshot that avoids this code duplication.
-
- @SuppressWarnings("unchecked")
- RegisteredPriorityQueueStateBackendMetaInfo<T> castedMetaInfo =
- (RegisteredPriorityQueueStateBackendMetaInfo<T>) stateInfo.metaInfo;
-
- TypeSerializer<T> previousElementSerializer = castedMetaInfo.getPreviousElementSerializer();
-
- if (previousElementSerializer != byteOrderedElementSerializer) {
- TypeSerializerSchemaCompatibility<T> compatibilityResult =
- castedMetaInfo.updateElementSerializer(byteOrderedElementSerializer);
-
- // Since priority queue elements are written into RocksDB
- // as keys prefixed with the key group and namespace, we do not support
- // migrating them. Therefore, here we only check for incompatibility.
- if (compatibilityResult.isIncompatible()) {
- throw new FlinkRuntimeException(
- new StateMigrationException("The new priority queue serializer must not be incompatible."));
- }
-
- // update meta info with new serializer
- stateInfo = new RocksDbKvStateInfo(
- stateInfo.columnFamilyHandle,
- new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer));
- kvStateInformation.put(stateName, stateInfo);
- }
- }
-
- return stateInfo;
- }
-
@Override
public boolean requiresLegacySynchronousTimerSnapshots() {
return priorityQueueFactory instanceof HeapPriorityQueueSetFactory;
}
/** Rocks DB specific information about the k/v states. */
- public static class RocksDbKvStateInfo {
+ public static class RocksDbKvStateInfo implements AutoCloseable{
public final ColumnFamilyHandle columnFamilyHandle;
public final RegisteredStateMetaInfoBase metaInfo;
- private RocksDbKvStateInfo(
+ public RocksDbKvStateInfo(
ColumnFamilyHandle columnFamilyHandle,
RegisteredStateMetaInfoBase metaInfo) {
this.columnFamilyHandle = columnFamilyHandle;
this.metaInfo = metaInfo;
}
+
+ @Override
+ public void close() throws Exception {
+ this.columnFamilyHandle.close();
+ }
}
@VisibleForTesting
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java
new file mode 100644
index 0000000..c9a03a2
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.contrib.streaming.state.restore.AbstractRocksDBRestoreOperation;
+import org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation;
+import org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation;
+import org.apache.flink.contrib.streaming.state.restore.RocksDBNoneRestoreOperation;
+import org.apache.flink.contrib.streaming.state.restore.RocksDBRestoreResult;
+import org.apache.flink.contrib.streaming.state.snapshot.RocksDBSnapshotStrategyBase;
+import org.apache.flink.contrib.streaming.state.snapshot.RocksFullSnapshotStrategy;
+import org.apache.flink.contrib.streaming.state.snapshot.RocksIncrementalSnapshotStrategy;
+import org.apache.flink.contrib.streaming.state.ttl.RocksDbTtlCompactFiltersManager;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.query.TaskKvStateRegistry;
+import org.apache.flink.runtime.state.AbstractKeyedStateBackendBuilder;
+import org.apache.flink.runtime.state.BackendBuildingException;
+import org.apache.flink.runtime.state.IncrementalKeyedStateHandle;
+import org.apache.flink.runtime.state.IncrementalLocalKeyedStateHandle;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.LocalRecoveryConfig;
+import org.apache.flink.runtime.state.PriorityQueueSetFactory;
+import org.apache.flink.runtime.state.StateHandleID;
+import org.apache.flink.runtime.state.StreamCompressionDecorator;
+import org.apache.flink.runtime.state.heap.HeapPriorityQueueSetFactory;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.ResourceGuard;
+
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.WriteOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.function.Function;
+
+/**
+ * Builder class for {@link RocksDBKeyedStateBackend} which handles all necessary initializations and clean ups.
+ *
+ * @param <K> The data type that the key serializer serializes.
+ */
+public class RocksDBKeyedStateBackendBuilder<K> extends AbstractKeyedStateBackendBuilder<K> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RocksDBKeyedStateBackendBuilder.class);
+ public static final String DB_INSTANCE_DIR_STRING = "db";
+
+ /** String that identifies the operator that owns this backend. */
+ private final String operatorIdentifier;
+ private final RocksDBStateBackend.PriorityQueueStateType priorityQueueStateType;
+ /** The configuration of local recovery. */
+ private final LocalRecoveryConfig localRecoveryConfig;
+
+ /** Factory function to create column family options from state name. */
+ private final Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory;
+
+ /** The DB options from the options factory. */
+ private final DBOptions dbOptions;
+
+ /** Path where this configured instance stores its data directory. */
+ private final File instanceBasePath;
+
+ /** Path where this configured instance stores its RocksDB database. */
+ private final File instanceRocksDBPath;
+
+ private final MetricGroup metricGroup;
+
+ /** True if incremental checkpointing is enabled. */
+ private boolean enableIncrementalCheckpointing;
+ /** True if ttl compaction filter is enabled. */
+ private boolean enableTtlCompactionFilter;
+ private RocksDBNativeMetricOptions nativeMetricOptions;
+ private int numberOfTransferingThreads;
+
+ private RocksDB injectedTestDB; // for testing
+ private ColumnFamilyHandle injectedDefaultColumnFamilyHandle; // for testing
+
+ public RocksDBKeyedStateBackendBuilder(
+ String operatorIdentifier,
+ ClassLoader userCodeClassLoader,
+ File instanceBasePath,
+ DBOptions dbOptions,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+ TaskKvStateRegistry kvStateRegistry,
+ TypeSerializer<K> keySerializer,
+ int numberOfKeyGroups,
+ KeyGroupRange keyGroupRange,
+ ExecutionConfig executionConfig,
+ LocalRecoveryConfig localRecoveryConfig,
+ RocksDBStateBackend.PriorityQueueStateType priorityQueueStateType,
+ TtlTimeProvider ttlTimeProvider,
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> stateHandles,
+ StreamCompressionDecorator keyGroupCompressionDecorator) {
+
+ super(
+ kvStateRegistry,
+ keySerializer,
+ userCodeClassLoader,
+ numberOfKeyGroups,
+ keyGroupRange,
+ executionConfig,
+ ttlTimeProvider,
+ stateHandles,
+ keyGroupCompressionDecorator
+ );
+
+ this.operatorIdentifier = operatorIdentifier;
+ this.priorityQueueStateType = priorityQueueStateType;
+ this.localRecoveryConfig = localRecoveryConfig;
+ // ensure that we use the right merge operator, because other code relies on this
+ this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);
+ this.dbOptions = dbOptions;
+ this.instanceBasePath = instanceBasePath;
+ this.instanceRocksDBPath = new File(instanceBasePath, DB_INSTANCE_DIR_STRING);
+ this.metricGroup = metricGroup;
+ this.enableIncrementalCheckpointing = false;
+ this.nativeMetricOptions = new RocksDBNativeMetricOptions();
+ this.numberOfTransferingThreads = RocksDBOptions.CHECKPOINT_TRANSFER_THREAD_NUM.defaultValue();
+ }
+
+ @VisibleForTesting
+ RocksDBKeyedStateBackendBuilder(
+ String operatorIdentifier,
+ ClassLoader userCodeClassLoader,
+ File instanceBasePath,
+ DBOptions dbOptions,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+ TaskKvStateRegistry kvStateRegistry,
+ TypeSerializer<K> keySerializer,
+ int numberOfKeyGroups,
+ KeyGroupRange keyGroupRange,
+ ExecutionConfig executionConfig,
+ LocalRecoveryConfig localRecoveryConfig,
+ RocksDBStateBackend.PriorityQueueStateType priorityQueueStateType,
+ TtlTimeProvider ttlTimeProvider,
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> stateHandles,
+ StreamCompressionDecorator keyGroupCompressionDecorator,
+ RocksDB injectedTestDB,
+ ColumnFamilyHandle injectedDefaultColumnFamilyHandle) {
+ this(
+ operatorIdentifier,
+ userCodeClassLoader,
+ instanceBasePath,
+ dbOptions,
+ columnFamilyOptionsFactory,
+ kvStateRegistry,
+ keySerializer,
+ numberOfKeyGroups,
+ keyGroupRange,
+ executionConfig,
+ localRecoveryConfig,
+ priorityQueueStateType,
+ ttlTimeProvider,
+ metricGroup,
+ stateHandles,
+ keyGroupCompressionDecorator
+ );
+ this.injectedTestDB = injectedTestDB;
+ this.injectedDefaultColumnFamilyHandle = injectedDefaultColumnFamilyHandle;
+ }
+
+ public RocksDBKeyedStateBackendBuilder<K> setEnableIncrementalCheckpointing(boolean enableIncrementalCheckpointing) {
+ this.enableIncrementalCheckpointing = enableIncrementalCheckpointing;
+ return this;
+ }
+
+ public RocksDBKeyedStateBackendBuilder<K> setEnableTtlCompactionFilter (boolean enableTtlCompactionFilter) {
+ this.enableTtlCompactionFilter = enableTtlCompactionFilter;
+ return this;
+ }
+
+ public RocksDBKeyedStateBackendBuilder<K> setNativeMetricOptions(RocksDBNativeMetricOptions nativeMetricOptions) {
+ this.nativeMetricOptions = nativeMetricOptions;
+ return this;
+ }
+
+ public RocksDBKeyedStateBackendBuilder<K> setNumberOfTransferingThreads(int numberOfTransferingThreads) {
+ this.numberOfTransferingThreads = numberOfTransferingThreads;
+ return this;
+ }
+
+ private static void checkAndCreateDirectory(File directory) throws IOException {
+ if (directory.exists()) {
+ if (!directory.isDirectory()) {
+ throw new IOException("Not a directory: " + directory);
+ }
+ } else if (!directory.mkdirs()) {
+ throw new IOException(String.format("Could not create RocksDB data directory at %s.", directory));
+ }
+ }
+
+ public RocksDBKeyedStateBackend<K> build() throws BackendBuildingException {
+ RocksDBWriteBatchWrapper writeBatchWrapper = null;
+ ColumnFamilyHandle defaultColumnFamilyHandle = null;
+ RocksDBNativeMetricMonitor nativeMetricMonitor = null;
+ CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
+ //The write options to use in the states. We disable write ahead logging.
+ WriteOptions writeOptions = new WriteOptions().setDisableWAL(true);
+ LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation = new LinkedHashMap<>();
+ RocksDB db = null;
+ AbstractRocksDBRestoreOperation restoreOperation = null;
+ RocksDbTtlCompactFiltersManager ttlCompactFiltersManager =
+ new RocksDbTtlCompactFiltersManager(enableTtlCompactionFilter);
+
+ ResourceGuard rocksDBResourceGuard = new ResourceGuard();
+ SnapshotStrategy<K> snapshotStrategy;
+ PriorityQueueSetFactory priorityQueueFactory;
+ RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder;
+ // Number of bytes required to prefix the key groups.
+ int keyGroupPrefixBytes = RocksDBKeySerializationUtils.computeRequiredBytesInKeyGroupPrefix(numberOfKeyGroups);
+ try {
+ // Variables for snapshot strategy when incremental checkpoint is enabled
+ UUID backendUID = UUID.randomUUID();
+ SortedMap<Long, Set<StateHandleID>> materializedSstFiles = new TreeMap<>();
+ long lastCompletedCheckpointId = -1L;
+ if (injectedTestDB != null) {
+ db = injectedTestDB;
+ defaultColumnFamilyHandle = injectedDefaultColumnFamilyHandle;
+ nativeMetricMonitor = nativeMetricOptions.isEnabled() ?
+ new RocksDBNativeMetricMonitor(nativeMetricOptions, metricGroup, db) : null;
+ } else {
+ prepareDirectories();
+ restoreOperation = getRocksDBRestoreOperation(
+ keyGroupPrefixBytes, cancelStreamRegistry, kvStateInformation, ttlCompactFiltersManager);
+ RocksDBRestoreResult restoreResult = restoreOperation.restore();
+ db = restoreResult.getDb();
+ defaultColumnFamilyHandle = restoreResult.getDefaultColumnFamilyHandle();
+ nativeMetricMonitor = restoreResult.getNativeMetricMonitor();
+ if (restoreOperation instanceof RocksDBIncrementalRestoreOperation) {
+ backendUID = restoreResult.getBackendUID();
+ materializedSstFiles = restoreResult.getRestoredSstFiles();
+ lastCompletedCheckpointId = restoreResult.getLastCompletedCheckpointId();
+ }
+ }
+
+ writeBatchWrapper = new RocksDBWriteBatchWrapper(db, writeOptions);
+ // it is important that we only create the key builder after the restore, and not before;
+ // restore operations may reconfigure the key serializer, so accessing the key serializer
+ // only now we can be certain that the key serializer used in the builder is final.
+ sharedRocksKeyBuilder = new RocksDBSerializedCompositeKeyBuilder<>(
+ keySerializerProvider.currentSchemaSerializer(),
+ keyGroupPrefixBytes,
+ 32);
+ // init snapshot strategy after db is assured to be initialized
+ snapshotStrategy = initializeSavepointAndCheckpointStrategies(cancelStreamRegistry, rocksDBResourceGuard,
+ kvStateInformation, keyGroupPrefixBytes, db, backendUID, materializedSstFiles, lastCompletedCheckpointId);
+ // init priority queue factory
+ priorityQueueFactory = initPriorityQueueFactory(keyGroupPrefixBytes, kvStateInformation, db,
+ writeBatchWrapper, nativeMetricMonitor);
+ } catch (Throwable e) {
+ // Do clean up
+ List<ColumnFamilyOptions> columnFamilyOptions = new ArrayList<>(kvStateInformation.values().size());
+ IOUtils.closeQuietly(cancelStreamRegistry);
+ IOUtils.closeQuietly(writeBatchWrapper);
+ RocksDBOperationUtils.addColumnFamilyOptionsToCloseLater(columnFamilyOptions, defaultColumnFamilyHandle);
+ IOUtils.closeQuietly(defaultColumnFamilyHandle);
+ IOUtils.closeQuietly(nativeMetricMonitor);
+ for (RocksDBKeyedStateBackend.RocksDbKvStateInfo kvStateInfo : kvStateInformation.values()) {
+ RocksDBOperationUtils.addColumnFamilyOptionsToCloseLater(columnFamilyOptions, kvStateInfo.columnFamilyHandle);
+ IOUtils.closeQuietly(kvStateInfo.columnFamilyHandle);
+ }
+ IOUtils.closeQuietly(db);
+ // it's possible that db has been initialized but later restore steps failed
+ IOUtils.closeQuietly(restoreOperation);
+ IOUtils.closeAllQuietly(columnFamilyOptions);
+ IOUtils.closeQuietly(dbOptions);
+ IOUtils.closeQuietly(writeOptions);
+ ttlCompactFiltersManager.disposeAndClearRegisteredCompactionFactories();
+ kvStateInformation.clear();
+ try {
+ FileUtils.deleteDirectory(instanceBasePath);
+ } catch (Exception ex) {
+ LOG.warn("Failed to instance base path for RocksDB: " + instanceBasePath, ex);
+ }
+ // Log and rethrow
+ if (e instanceof BackendBuildingException) {
+ throw (BackendBuildingException) e;
+ } else {
+ String errMsg = "Caught unexpected exception.";
+ LOG.error(errMsg, e);
+ throw new BackendBuildingException(errMsg, e);
+ }
+ }
+ return new RocksDBKeyedStateBackend<>(
+ this.operatorIdentifier,
+ this.userCodeClassLoader,
+ this.instanceBasePath,
+ this.dbOptions,
+ columnFamilyOptionsFactory,
+ this.kvStateRegistry,
+ this.keySerializerProvider,
+ this.numberOfKeyGroups,
+ this.keyGroupRange,
+ this.executionConfig,
+ this.numberOfTransferingThreads,
+ this.ttlTimeProvider,
+ db,
+ kvStateInformation,
+ keyGroupPrefixBytes,
+ cancelStreamRegistry,
+ this.keyGroupCompressionDecorator,
+ rocksDBResourceGuard,
+ snapshotStrategy.checkpointSnapshotStrategy,
+ snapshotStrategy.savepointSnapshotStrategy,
+ writeBatchWrapper,
+ defaultColumnFamilyHandle,
+ nativeMetricMonitor,
+ sharedRocksKeyBuilder,
+ priorityQueueFactory,
+ ttlCompactFiltersManager
+ );
+ }
+
+ private AbstractRocksDBRestoreOperation<K> getRocksDBRestoreOperation(
+ int keyGroupPrefixBytes,
+ CloseableRegistry cancelStreamRegistry,
+ LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
+ RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
+ if (restoreStateHandles.isEmpty()) {
+ return new RocksDBNoneRestoreOperation<>(
+ keyGroupRange,
+ keyGroupPrefixBytes,
+ numberOfTransferingThreads,
+ cancelStreamRegistry,
+ userCodeClassLoader,
+ kvStateInformation,
+ keySerializerProvider,
+ instanceBasePath,
+ instanceRocksDBPath,
+ dbOptions,
+ columnFamilyOptionsFactory,
+ nativeMetricOptions,
+ metricGroup,
+ restoreStateHandles,
+ ttlCompactFiltersManager,
+ ttlTimeProvider);
+ }
+ KeyedStateHandle firstStateHandle = restoreStateHandles.iterator().next();
+ boolean isIncrementalStateHandle = (firstStateHandle instanceof IncrementalKeyedStateHandle)
+ || (firstStateHandle instanceof IncrementalLocalKeyedStateHandle);
+ if (isIncrementalStateHandle) {
+ return new RocksDBIncrementalRestoreOperation<>(
+ operatorIdentifier,
+ keyGroupRange,
+ keyGroupPrefixBytes,
+ numberOfTransferingThreads,
+ cancelStreamRegistry,
+ userCodeClassLoader,
+ kvStateInformation,
+ keySerializerProvider,
+ instanceBasePath,
+ instanceRocksDBPath,
+ dbOptions,
+ columnFamilyOptionsFactory,
+ nativeMetricOptions,
+ metricGroup,
+ restoreStateHandles,
+ ttlCompactFiltersManager,
+ ttlTimeProvider);
+ } else {
+ return new RocksDBFullRestoreOperation<>(
+ keyGroupRange,
+ keyGroupPrefixBytes,
+ numberOfTransferingThreads,
+ cancelStreamRegistry,
+ userCodeClassLoader,
+ kvStateInformation,
+ keySerializerProvider,
+ instanceBasePath,
+ instanceRocksDBPath,
+ dbOptions,
+ columnFamilyOptionsFactory,
+ nativeMetricOptions,
+ metricGroup,
+ restoreStateHandles,
+ ttlCompactFiltersManager,
+ ttlTimeProvider);
+ }
+ }
+
+ private SnapshotStrategy<K> initializeSavepointAndCheckpointStrategies(
+ CloseableRegistry cancelStreamRegistry,
+ ResourceGuard rocksDBResourceGuard,
+ LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
+ int keyGroupPrefixBytes,
+ RocksDB db,
+ UUID backendUID,
+ SortedMap<Long, Set<StateHandleID>> materializedSstFiles,
+ long lastCompletedCheckpointId) {
+ RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy = new RocksFullSnapshotStrategy<>(
+ db,
+ rocksDBResourceGuard,
+ keySerializer,
+ kvStateInformation,
+ keyGroupRange,
+ keyGroupPrefixBytes,
+ localRecoveryConfig,
+ cancelStreamRegistry,
+ keyGroupCompressionDecorator);
+ RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy;
+ if (enableIncrementalCheckpointing) {
+ // TODO eventually we might want to separate savepoint and snapshot strategy, i.e. having 2 strategies.
+ checkpointSnapshotStrategy = new RocksIncrementalSnapshotStrategy<>(
+ db,
+ rocksDBResourceGuard,
+ keySerializer,
+ kvStateInformation,
+ keyGroupRange,
+ keyGroupPrefixBytes,
+ localRecoveryConfig,
+ cancelStreamRegistry,
+ instanceBasePath,
+ backendUID,
+ materializedSstFiles,
+ lastCompletedCheckpointId,
+ numberOfTransferingThreads);
+ } else {
+ checkpointSnapshotStrategy = savepointSnapshotStrategy;
+ }
+ return new SnapshotStrategy<>(checkpointSnapshotStrategy, savepointSnapshotStrategy);
+ }
+
+ private PriorityQueueSetFactory initPriorityQueueFactory(
+ int keyGroupPrefixBytes,
+ Map<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
+ RocksDB db,
+ RocksDBWriteBatchWrapper writeBatchWrapper,
+ RocksDBNativeMetricMonitor nativeMetricMonitor) {
+ PriorityQueueSetFactory priorityQueueFactory;
+ switch (priorityQueueStateType) {
+ case HEAP:
+ priorityQueueFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
+ break;
+ case ROCKSDB:
+ priorityQueueFactory = new RocksDBPriorityQueueSetFactory(
+ keyGroupRange,
+ keyGroupPrefixBytes,
+ numberOfKeyGroups,
+ kvStateInformation,
+ db,
+ writeBatchWrapper,
+ nativeMetricMonitor,
+ columnFamilyOptionsFactory
+ );
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown priority queue state type: " + priorityQueueStateType);
+ }
+ return priorityQueueFactory;
+ }
+
+ private void prepareDirectories() throws IOException {
+ checkAndCreateDirectory(instanceBasePath);
+ if (instanceRocksDBPath.exists()) {
+ // Clear the base directory when the backend is created
+ // in case something crashed and the backend never reached dispose()
+ FileUtils.deleteDirectory(instanceBasePath);
+ }
+ }
+
+ static final class SnapshotStrategy<K> {
+ final RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy;
+ final RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy;
+
+ SnapshotStrategy(RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
+ RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy) {
+ this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
+ this.savepointSnapshotStrategy = savepointSnapshotStrategy;
+ }
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java
index 899fb8a..ebe8cbf 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java
@@ -217,7 +217,7 @@ class RocksDBMapState<K, N, UK, UV>
@Override
public void clear() {
try {
- try (RocksIteratorWrapper iterator = RocksDBKeyedStateBackend.getRocksIterator(backend.db, columnFamily);
+ try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(backend.db, columnFamily);
RocksDBWriteBatchWrapper rocksDBWriteBatchWrapper = new RocksDBWriteBatchWrapper(backend.db, backend.getWriteOptions())) {
final byte[] keyPrefixBytes = serializeCurrentKeyWithGroupAndNamespace();
@@ -540,7 +540,7 @@ class RocksDBMapState<K, N, UK, UV>
// use try-with-resources to ensure RocksIterator can be release even some runtime exception
// occurred in the below code block.
- try (RocksIteratorWrapper iterator = RocksDBKeyedStateBackend.getRocksIterator(db, columnFamily)) {
+ try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnFamily)) {
/*
* The iteration starts from the prefix bytes at the first loading. After #nextEntry() is called,
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBNativeMetricMonitor.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBNativeMetricMonitor.java
index 5e60cee..a42ef57 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBNativeMetricMonitor.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBNativeMetricMonitor.java
@@ -53,10 +53,10 @@ public class RocksDBNativeMetricMonitor implements Closeable {
@GuardedBy("lock")
private RocksDB rocksDB;
- RocksDBNativeMetricMonitor(
- RocksDB rocksDB,
+ public RocksDBNativeMetricMonitor(
@Nonnull RocksDBNativeMetricOptions options,
- @Nonnull MetricGroup metricGroup
+ @Nonnull MetricGroup metricGroup,
+ @Nonnull RocksDB rocksDB
) {
this.options = options;
this.metricGroup = metricGroup;
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOperationUtils.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOperationUtils.java
new file mode 100644
index 0000000..b183426
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOperationUtils.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state;
+
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.contrib.streaming.state.ttl.RocksDbTtlCompactFiltersManager;
+import org.apache.flink.runtime.state.RegisteredStateMetaInfoBase;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.Preconditions;
+
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import static org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME;
+
+/**
+ * Utils for RocksDB Operations.
+ */
+public class RocksDBOperationUtils {
+ public static RocksDB openDB(
+ String path,
+ List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
+ List<ColumnFamilyHandle> stateColumnFamilyHandles,
+ ColumnFamilyOptions columnFamilyOptions,
+ DBOptions dbOptions) throws IOException {
+ List<ColumnFamilyDescriptor> columnFamilyDescriptors =
+ new ArrayList<>(1 + stateColumnFamilyDescriptors.size());
+
+ // we add the required descriptor for the default CF in FIRST position, see
+ // https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
+ columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
+ columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);
+
+ RocksDB dbRef;
+
+ try {
+ dbRef = RocksDB.open(
+ Preconditions.checkNotNull(dbOptions),
+ Preconditions.checkNotNull(path),
+ columnFamilyDescriptors,
+ stateColumnFamilyHandles);
+ } catch (RocksDBException e) {
+ IOUtils.closeQuietly(columnFamilyOptions);
+ columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));
+ throw new IOException("Error while opening RocksDB instance.", e);
+ }
+
+ // requested + default CF
+ Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(),
+ "Not all requested column family handles have been created");
+ return dbRef;
+ }
+
+ public static ColumnFamilyDescriptor createColumnFamilyDescriptor(String stateName, ColumnFamilyOptions columnOptions) {
+ byte[] nameBytes = stateName.getBytes(ConfigConstants.DEFAULT_CHARSET);
+ Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes),
+ "The chosen state name 'default' collides with the name of the default column family!");
+
+ return new ColumnFamilyDescriptor(nameBytes, columnOptions);
+ }
+
+ public static ColumnFamilyHandle createColumnFamily(ColumnFamilyDescriptor columnDescriptor, RocksDB db) {
+ try {
+ return db.createColumnFamily(columnDescriptor);
+ } catch (RocksDBException e) {
+ IOUtils.closeQuietly(columnDescriptor.getOptions());
+ throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e);
+ }
+ }
+
+ public static ColumnFamilyHandle createColumnFamily(
+ String stateName,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+ RocksDB db) {
+ ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, stateName);
+ return createColumnFamily(createColumnFamilyDescriptor(stateName, options), db);
+ }
+
+ public static RocksIteratorWrapper getRocksIterator(RocksDB db) {
+ return new RocksIteratorWrapper(db.newIterator());
+ }
+
+ public static RocksIteratorWrapper getRocksIterator(
+ RocksDB db,
+ ColumnFamilyHandle columnFamilyHandle) {
+ return new RocksIteratorWrapper(db.newIterator(columnFamilyHandle));
+ }
+
+ public static void registerKvStateInformation(
+ Map<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
+ RocksDBNativeMetricMonitor nativeMetricMonitor,
+ String columnFamilyName,
+ RocksDBKeyedStateBackend.RocksDbKvStateInfo registeredColumn) {
+ kvStateInformation.put(columnFamilyName, registeredColumn);
+
+ if (nativeMetricMonitor != null) {
+ nativeMetricMonitor.registerColumnFamily(columnFamilyName, registeredColumn.columnFamilyHandle);
+ }
+ }
+
+ /**
+ * Creates a state info from a new meta info to use with a k/v state.
+ */
+ public static RocksDBKeyedStateBackend.RocksDbKvStateInfo createStateInfo(
+ RegisteredStateMetaInfoBase metaInfoBase,
+ RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
+ TtlTimeProvider ttlTimeProvider,
+ RocksDB db,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory) {
+ ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, metaInfoBase.getName());
+ ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(ttlTimeProvider, metaInfoBase, options);
+ ColumnFamilyDescriptor columnFamilyDescriptor = createColumnFamilyDescriptor(metaInfoBase.getName(), options);
+ return new RocksDBKeyedStateBackend.RocksDbKvStateInfo(createColumnFamily(columnFamilyDescriptor, db), metaInfoBase);
+ }
+
+ public static ColumnFamilyOptions createColumnFamilyOptions(
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+ String stateName) {
+ // ensure that we use the right merge operator, because other code relies on this
+ return columnFamilyOptionsFactory.apply(stateName).setMergeOperatorName(MERGE_OPERATOR_NAME);
+ }
+
+ public static void addColumnFamilyOptionsToCloseLater(
+ List<ColumnFamilyOptions> columnFamilyOptions, ColumnFamilyHandle columnFamilyHandle) {
+ try {
+ if (columnFamilyHandle != null && columnFamilyHandle.getDescriptor() != null) {
+ columnFamilyOptions.add(columnFamilyHandle.getDescriptor().getOptions());
+ }
+ } catch (RocksDBException e) {
+ // ignore
+ }
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBPriorityQueueSetFactory.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBPriorityQueueSetFactory.java
new file mode 100644
index 0000000..1eb0f47
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBPriorityQueueSetFactory.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataOutputSerializer;
+import org.apache.flink.runtime.state.KeyExtractorFunction;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyGroupedInternalPriorityQueue;
+import org.apache.flink.runtime.state.Keyed;
+import org.apache.flink.runtime.state.PriorityComparable;
+import org.apache.flink.runtime.state.PriorityComparator;
+import org.apache.flink.runtime.state.PriorityQueueSetFactory;
+import org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo;
+import org.apache.flink.runtime.state.heap.HeapPriorityQueueElement;
+import org.apache.flink.runtime.state.heap.KeyGroupPartitionedPriorityQueue;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.StateMigrationException;
+
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.RocksDB;
+
+import javax.annotation.Nonnull;
+
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * Encapsulates the logic and resources in connection with creating priority
+ * queue state structures, for RocksDB backend.
+ */
+public class RocksDBPriorityQueueSetFactory implements PriorityQueueSetFactory {
+
+ /**
+ * Default cache size per key-group.
+ */
+ private static final int DEFAULT_CACHES_SIZE = 128; //TODO make this configurable
+
+ /**
+ * A shared buffer to serialize elements for the priority queue.
+ */
+ @Nonnull
+ private final DataOutputSerializer sharedElementOutView;
+
+ /**
+ * A shared buffer to de-serialize elements for the priority queue.
+ */
+ @Nonnull
+ private final DataInputDeserializer sharedElementInView;
+
+ private final KeyGroupRange keyGroupRange;
+ private final int keyGroupPrefixBytes;
+ private final int numberOfKeyGroups;
+ private final Map<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation;
+ private final RocksDB db;
+ private final RocksDBWriteBatchWrapper writeBatchWrapper;
+ private final RocksDBNativeMetricMonitor nativeMetricMonitor;
+ private final Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory;
+
+ RocksDBPriorityQueueSetFactory(
+ KeyGroupRange keyGroupRange,
+ int keyGroupPrefixBytes,
+ int numberOfKeyGroups,
+ Map<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
+ RocksDB db,
+ RocksDBWriteBatchWrapper writeBatchWrapper,
+ RocksDBNativeMetricMonitor nativeMetricMonitor,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory) {
+ this.keyGroupRange = keyGroupRange;
+ this.keyGroupPrefixBytes = keyGroupPrefixBytes;
+ this.numberOfKeyGroups = numberOfKeyGroups;
+ this.kvStateInformation = kvStateInformation;
+ this.db = db;
+ this.writeBatchWrapper = writeBatchWrapper;
+ this.nativeMetricMonitor = nativeMetricMonitor;
+ this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
+ this.sharedElementOutView = new DataOutputSerializer(128);
+ this.sharedElementInView = new DataInputDeserializer();
+ }
+
+ @Nonnull
+ @Override
+ public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T>
+ create(@Nonnull String stateName, @Nonnull TypeSerializer<T> byteOrderedElementSerializer) {
+
+ final RocksDBKeyedStateBackend.RocksDbKvStateInfo stateCFHandle =
+ tryRegisterPriorityQueueMetaInfo(stateName, byteOrderedElementSerializer);
+
+ final ColumnFamilyHandle columnFamilyHandle = stateCFHandle.columnFamilyHandle;
+
+ return new KeyGroupPartitionedPriorityQueue<>(
+ KeyExtractorFunction.forKeyedObjects(),
+ PriorityComparator.forPriorityComparableObjects(),
+ new KeyGroupPartitionedPriorityQueue.PartitionQueueSetFactory<T, RocksDBCachingPriorityQueueSet<T>>() {
+ @Nonnull
+ @Override
+ public RocksDBCachingPriorityQueueSet<T> create(
+ int keyGroupId,
+ int numKeyGroups,
+ @Nonnull KeyExtractorFunction<T> keyExtractor,
+ @Nonnull PriorityComparator<T> elementPriorityComparator) {
+ TreeOrderedSetCache orderedSetCache = new TreeOrderedSetCache(DEFAULT_CACHES_SIZE);
+ return new RocksDBCachingPriorityQueueSet<>(
+ keyGroupId,
+ keyGroupPrefixBytes,
+ db,
+ columnFamilyHandle,
+ byteOrderedElementSerializer,
+ sharedElementOutView,
+ sharedElementInView,
+ writeBatchWrapper,
+ orderedSetCache
+ );
+ }
+ },
+ keyGroupRange,
+ numberOfKeyGroups);
+ }
+
+ @Nonnull
+ private <T> RocksDBKeyedStateBackend.RocksDbKvStateInfo tryRegisterPriorityQueueMetaInfo(
+ @Nonnull String stateName,
+ @Nonnull TypeSerializer<T> byteOrderedElementSerializer) {
+
+ RocksDBKeyedStateBackend.RocksDbKvStateInfo stateInfo = kvStateInformation.get(stateName);
+
+ if (stateInfo == null) {
+ // Currently this class is for timer service and TTL feature is not applicable here,
+ // so no need to register compact filter when creating column family
+ final ColumnFamilyHandle columnFamilyHandle =
+ RocksDBOperationUtils.createColumnFamily(stateName, columnFamilyOptionsFactory, this.db);
+ RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo =
+ new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
+
+ stateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(columnFamilyHandle, metaInfo);
+ RocksDBOperationUtils.registerKvStateInformation(kvStateInformation, nativeMetricMonitor, stateName, stateInfo);
+ } else {
+ // TODO we implement the simple way of supporting the current functionality, mimicking keyed state
+ // because this should be reworked in FLINK-9376 and then we should have a common algorithm over
+ // StateMetaInfoSnapshot that avoids this code duplication.
+
+ @SuppressWarnings("unchecked")
+ RegisteredPriorityQueueStateBackendMetaInfo<T> castedMetaInfo =
+ (RegisteredPriorityQueueStateBackendMetaInfo<T>) stateInfo.metaInfo;
+
+ TypeSerializer<T> previousElementSerializer = castedMetaInfo.getPreviousElementSerializer();
+
+ if (previousElementSerializer != byteOrderedElementSerializer) {
+ TypeSerializerSchemaCompatibility<T> compatibilityResult =
+ castedMetaInfo.updateElementSerializer(byteOrderedElementSerializer);
+
+ // Since priority queue elements are written into RocksDB
+ // as keys prefixed with the key group and namespace, we do not support
+ // migrating them. Therefore, here we only check for incompatibility.
+ if (compatibilityResult.isIncompatible()) {
+ throw new FlinkRuntimeException(
+ new StateMigrationException("The new priority queue serializer must not be incompatible."));
+ }
+
+ // update meta info with new serializer
+ stateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(
+ stateInfo.columnFamilyHandle,
+ new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer));
+ kvStateInformation.put(stateName, stateInfo);
+ }
+ }
+
+ return stateInfo;
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
index df82880..5db2e9b 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
@@ -18,6 +18,7 @@
package org.apache.flink.contrib.streaming.state;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.configuration.CheckpointingOptions;
@@ -34,9 +35,13 @@ import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
import org.apache.flink.runtime.state.ConfigurableStateBackend;
import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.LocalRecoveryConfig;
import org.apache.flink.runtime.state.OperatorStateBackend;
+import org.apache.flink.runtime.state.SnappyStreamCompressionDecorator;
import org.apache.flink.runtime.state.StateBackend;
+import org.apache.flink.runtime.state.StreamCompressionDecorator;
+import org.apache.flink.runtime.state.UncompressedStreamCompressionDecorator;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.apache.flink.util.AbstractID;
@@ -50,6 +55,7 @@ import org.rocksdb.RocksDB;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.File;
@@ -58,6 +64,7 @@ import java.lang.reflect.Field;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.Random;
import java.util.UUID;
@@ -442,15 +449,16 @@ public class RocksDBStateBackend extends AbstractStateBackend implements Configu
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
- Environment env,
- JobID jobID,
- String operatorIdentifier,
- TypeSerializer<K> keySerializer,
- int numberOfKeyGroups,
- KeyGroupRange keyGroupRange,
- TaskKvStateRegistry kvStateRegistry,
- TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) throws IOException {
+ Environment env,
+ JobID jobID,
+ String operatorIdentifier,
+ TypeSerializer<K> keySerializer,
+ int numberOfKeyGroups,
+ KeyGroupRange keyGroupRange,
+ TaskKvStateRegistry kvStateRegistry,
+ TtlTimeProvider ttlTimeProvider,
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> stateHandles) throws IOException {
// first, make sure that the RocksDB JNI library is loaded
// we do this explicitly here to have better error handling
@@ -469,25 +477,38 @@ public class RocksDBStateBackend extends AbstractStateBackend implements Configu
LocalRecoveryConfig localRecoveryConfig =
env.getTaskStateManager().createLocalRecoveryConfig();
- return new RocksDBKeyedStateBackend<>(
- operatorIdentifier,
- env.getUserClassLoader(),
- instanceBasePath,
- getDbOptions(),
- stateName -> getColumnOptions(),
- kvStateRegistry,
- keySerializer,
- numberOfKeyGroups,
- keyGroupRange,
- env.getExecutionConfig(),
- isIncrementalCheckpointsEnabled(),
- getNumberOfTransferingThreads(),
- localRecoveryConfig,
- priorityQueueStateType,
- ttlTimeProvider,
- enableTtlCompactionFilter.getOrDefault(TTL_COMPACT_FILTER_ENABLED.defaultValue()),
- getMemoryWatcherOptions(),
- metricGroup);
+ ExecutionConfig executionConfig = env.getExecutionConfig();
+ StreamCompressionDecorator keyGroupCompressionDecorator = getCompressionDecorator(executionConfig);
+ RocksDBKeyedStateBackendBuilder<K> builder = new RocksDBKeyedStateBackendBuilder<>(
+ operatorIdentifier,
+ env.getUserClassLoader(),
+ instanceBasePath,
+ getDbOptions(),
+ stateName -> getColumnOptions(),
+ kvStateRegistry,
+ keySerializer,
+ numberOfKeyGroups,
+ keyGroupRange,
+ executionConfig,
+ localRecoveryConfig,
+ priorityQueueStateType,
+ ttlTimeProvider,
+ metricGroup,
+ stateHandles,
+ keyGroupCompressionDecorator
+ ).setEnableIncrementalCheckpointing(isIncrementalCheckpointsEnabled())
+ .setEnableTtlCompactionFilter(isTtlCompactionFilterEnabled())
+ .setNumberOfTransferingThreads(getNumberOfTransferingThreads())
+ .setNativeMetricOptions(getMemoryWatcherOptions());
+ return builder.build();
+ }
+
+ public static StreamCompressionDecorator getCompressionDecorator(ExecutionConfig executionConfig) {
+ if (executionConfig != null && executionConfig.isUseSnapshotCompression()) {
+ return SnappyStreamCompressionDecorator.INSTANCE;
+ } else {
+ return UncompressedStreamCompressionDecorator.INSTANCE;
+ }
}
@Override
@@ -619,6 +640,13 @@ public class RocksDBStateBackend extends AbstractStateBackend implements Configu
}
/**
+ * Gets whether incremental checkpoints are enabled for this state backend.
+ */
+ public boolean isTtlCompactionFilterEnabled() {
+ return enableTtlCompactionFilter.getOrDefault(TTL_COMPACT_FILTER_ENABLED.defaultValue());
+ }
+
+ /**
* Enable compaction filter to cleanup state with TTL is enabled.
*
* <p>Note: User can still decide in state TTL configuration in state descriptor
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/AbstractRocksDBRestoreOperation.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/AbstractRocksDBRestoreOperation.java
new file mode 100644
index 0000000..9a29b62
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/AbstractRocksDBRestoreOperation.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state.restore;
+
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend.RocksDbKvStateInfo;
+import org.apache.flink.contrib.streaming.state.RocksDBNativeMetricMonitor;
+import org.apache.flink.contrib.streaming.state.RocksDBNativeMetricOptions;
+import org.apache.flink.contrib.streaming.state.RocksDBOperationUtils;
+import org.apache.flink.contrib.streaming.state.ttl.RocksDbTtlCompactFiltersManager;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.RegisteredStateMetaInfoBase;
+import org.apache.flink.runtime.state.StateSerializerProvider;
+import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.StateMigrationException;
+
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+
+import javax.annotation.Nonnull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * Base implementation of RocksDB restore operation.
+ *
+ * @param <K> The data type that the serializer serializes.
+ */
+public abstract class AbstractRocksDBRestoreOperation<K> implements RocksDBRestoreOperation, AutoCloseable {
+ protected final KeyGroupRange keyGroupRange;
+ protected final int keyGroupPrefixBytes;
+ protected final int numberOfTransferringThreads;
+ protected final CloseableRegistry cancelStreamRegistry;
+ protected final ClassLoader userCodeClassLoader;
+ protected final Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory;
+ protected final DBOptions dbOptions;
+ protected final Map<String, RocksDbKvStateInfo> kvStateInformation;
+ protected final File instanceBasePath;
+ protected final File instanceRocksDBPath;
+ protected final String dbPath;
+ protected List<ColumnFamilyHandle> columnFamilyHandles;
+ protected List<ColumnFamilyDescriptor> columnFamilyDescriptors;
+ protected final StateSerializerProvider<K> keySerializerProvider;
+ protected final RocksDBNativeMetricOptions nativeMetricOptions;
+ protected final MetricGroup metricGroup;
+ protected final Collection<KeyedStateHandle> restoreStateHandles;
+ // Current places to set compact filter into column family options:
+ // - Incremental restore
+ // - restore with rescaling
+ // - init from a certain sst: #createAndRegisterColumnFamilyDescriptors when prepare files, before db open
+ // - data ingestion after db open: #getOrRegisterStateColumnFamilyHandle before creating column family
+ // - restore without rescaling: #createAndRegisterColumnFamilyDescriptors when prepare files, before db open
+ // - Full restore
+ // - data ingestion after db open: #getOrRegisterStateColumnFamilyHandle before creating column family
+ protected final RocksDbTtlCompactFiltersManager ttlCompactFiltersManager;
+ protected final TtlTimeProvider ttlTimeProvider;
+
+ protected RocksDB db;
+ protected ColumnFamilyHandle defaultColumnFamilyHandle;
+ protected RocksDBNativeMetricMonitor nativeMetricMonitor;
+ protected boolean isKeySerializerCompatibilityChecked;
+
+ protected AbstractRocksDBRestoreOperation(
+ KeyGroupRange keyGroupRange,
+ int keyGroupPrefixBytes,
+ int numberOfTransferringThreads,
+ CloseableRegistry cancelStreamRegistry,
+ ClassLoader userCodeClassLoader,
+ Map<String, RocksDbKvStateInfo> kvStateInformation,
+ StateSerializerProvider<K> keySerializerProvider,
+ File instanceBasePath,
+ File instanceRocksDBPath,
+ DBOptions dbOptions,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+ RocksDBNativeMetricOptions nativeMetricOptions,
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> stateHandles,
+ @Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
+ TtlTimeProvider ttlTimeProvider) {
+ this.keyGroupRange = keyGroupRange;
+ this.keyGroupPrefixBytes = keyGroupPrefixBytes;
+ this.numberOfTransferringThreads = numberOfTransferringThreads;
+ this.cancelStreamRegistry = cancelStreamRegistry;
+ this.userCodeClassLoader = userCodeClassLoader;
+ this.kvStateInformation = kvStateInformation;
+ this.keySerializerProvider = keySerializerProvider;
+ this.instanceBasePath = instanceBasePath;
+ this.instanceRocksDBPath = instanceRocksDBPath;
+ this.dbPath = instanceRocksDBPath.getAbsolutePath();
+ this.dbOptions = dbOptions;
+ this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
+ this.nativeMetricOptions = nativeMetricOptions;
+ this.metricGroup = metricGroup;
+ this.restoreStateHandles = stateHandles;
+ this.ttlCompactFiltersManager = ttlCompactFiltersManager;
+ this.ttlTimeProvider = ttlTimeProvider;
+ this.columnFamilyHandles = new ArrayList<>(1);
+ this.columnFamilyDescriptors = Collections.emptyList();
+ }
+
+ public void openDB() throws IOException {
+ db = RocksDBOperationUtils.openDB(
+ dbPath,
+ columnFamilyDescriptors,
+ columnFamilyHandles,
+ RocksDBOperationUtils.createColumnFamilyOptions(columnFamilyOptionsFactory, "default"),
+ dbOptions);
+ // remove the default column family which is located at the first index
+ defaultColumnFamilyHandle = columnFamilyHandles.remove(0);
+ // init native metrics monitor if configured
+ nativeMetricMonitor = nativeMetricOptions.isEnabled() ?
+ new RocksDBNativeMetricMonitor(nativeMetricOptions, metricGroup, db) : null;
+ }
+
+ public RocksDB getDb() {
+ return this.db;
+ }
+
+ protected RocksDbKvStateInfo getOrRegisterStateColumnFamilyHandle(
+ ColumnFamilyHandle columnFamilyHandle,
+ StateMetaInfoSnapshot stateMetaInfoSnapshot) throws RocksDBException {
+
+ RocksDbKvStateInfo registeredStateMetaInfoEntry =
+ kvStateInformation.get(stateMetaInfoSnapshot.getName());
+
+ if (null == registeredStateMetaInfoEntry) {
+ // create a meta info for the state on restore;
+ // this allows us to retain the state in future snapshots even if it wasn't accessed
+ RegisteredStateMetaInfoBase stateMetaInfo =
+ RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot);
+ if (columnFamilyHandle == null) {
+ registeredStateMetaInfoEntry = RocksDBOperationUtils.createStateInfo(stateMetaInfo,
+ ttlCompactFiltersManager, ttlTimeProvider, db, columnFamilyOptionsFactory);
+ } else {
+ registeredStateMetaInfoEntry = new RocksDbKvStateInfo(columnFamilyHandle, stateMetaInfo);
+ }
+
+ RocksDBOperationUtils.registerKvStateInformation(
+ kvStateInformation,
+ nativeMetricMonitor,
+ stateMetaInfoSnapshot.getName(),
+ registeredStateMetaInfoEntry);
+ } else {
+ // TODO with eager state registration in place, check here for serializer migration strategies
+ }
+
+ return registeredStateMetaInfoEntry;
+ }
+
+ protected KeyedBackendSerializationProxy<K> readMetaData(DataInputView dataInputView)
+ throws IOException, StateMigrationException {
+ // isSerializerPresenceRequired flag is set to false, since for the RocksDB state backend,
+ // deserialization of state happens lazily during runtime; we depend on the fact
+ // that the new serializer for states could be compatible, and therefore the restore can continue
+ // without old serializers required to be present.
+ KeyedBackendSerializationProxy<K> serializationProxy =
+ new KeyedBackendSerializationProxy<>(userCodeClassLoader);
+ serializationProxy.read(dataInputView);
+ if (!isKeySerializerCompatibilityChecked) {
+ // check for key serializer compatibility; this also reconfigures the
+ // key serializer to be compatible, if it is required and is possible
+ TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
+ keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
+ if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
+ throw new StateMigrationException("The new key serializer must be compatible.");
+ }
+
+ isKeySerializerCompatibilityChecked = true;
+ }
+
+ return serializationProxy;
+ }
+
+ /**
+ * Necessary clean up iff restore operation failed.
+ */
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(defaultColumnFamilyHandle);
+ IOUtils.closeQuietly(nativeMetricMonitor);
+ IOUtils.closeQuietly(db);
+ // Making sure the already created column family options will be closed
+ columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));
+ }
+
+ @Override
+ public abstract RocksDBRestoreResult restore() throws Exception;
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBFullRestoreOperation.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBFullRestoreOperation.java
new file mode 100644
index 0000000..e9c8fe9
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBFullRestoreOperation.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state.restore;
+
+import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend.RocksDbKvStateInfo;
+import org.apache.flink.contrib.streaming.state.RocksDBNativeMetricOptions;
+import org.apache.flink.contrib.streaming.state.RocksDBWriteBatchWrapper;
+import org.apache.flink.contrib.streaming.state.ttl.RocksDbTtlCompactFiltersManager;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyGroupsStateHandle;
+import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.SnappyStreamCompressionDecorator;
+import org.apache.flink.runtime.state.StateSerializerProvider;
+import org.apache.flink.runtime.state.StreamCompressionDecorator;
+import org.apache.flink.runtime.state.UncompressedStreamCompressionDecorator;
+import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StateMigrationException;
+
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDBException;
+
+import javax.annotation.Nonnull;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import static org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.END_OF_KEY_GROUP_MARK;
+import static org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.clearMetaDataFollowsFlag;
+import static org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.hasMetaDataFollowsFlag;
+
+/**
+ * Encapsulates the process of restoring a RocksDB instance from a full snapshot.
+ */
+public class RocksDBFullRestoreOperation<K> extends AbstractRocksDBRestoreOperation<K> {
+ /**
+ * Current key-groups state handle from which we restore key-groups.
+ */
+ private KeyGroupsStateHandle currentKeyGroupsStateHandle;
+ /**
+ * Current input stream we obtained from currentKeyGroupsStateHandle.
+ */
+ private FSDataInputStream currentStateHandleInStream;
+ /**
+ * Current data input view that wraps currentStateHandleInStream.
+ */
+ private DataInputView currentStateHandleInView;
+ /**
+ * Current list of ColumnFamilyHandles for all column families we restore from currentKeyGroupsStateHandle.
+ */
+ private List<ColumnFamilyHandle> currentStateHandleKVStateColumnFamilies;
+ /**
+ * The compression decorator that was used for writing the state, as determined by the meta data.
+ */
+ private StreamCompressionDecorator keygroupStreamCompressionDecorator;
+
+ public RocksDBFullRestoreOperation(
+ KeyGroupRange keyGroupRange,
+ int keyGroupPrefixBytes,
+ int numberOfTransferringThreads,
+ CloseableRegistry cancelStreamRegistry,
+ ClassLoader userCodeClassLoader,
+ Map<String, RocksDbKvStateInfo> kvStateInformation,
+ StateSerializerProvider<K> keySerializerProvider,
+ File instanceBasePath,
+ File instanceRocksDBPath,
+ DBOptions dbOptions,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+ RocksDBNativeMetricOptions nativeMetricOptions,
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> restoreStateHandles,
+ @Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
+ TtlTimeProvider ttlTimeProvider) {
+ super(
+ keyGroupRange,
+ keyGroupPrefixBytes,
+ numberOfTransferringThreads,
+ cancelStreamRegistry,
+ userCodeClassLoader,
+ kvStateInformation,
+ keySerializerProvider,
+ instanceBasePath,
+ instanceRocksDBPath,
+ dbOptions,
+ columnFamilyOptionsFactory,
+ nativeMetricOptions,
+ metricGroup,
+ restoreStateHandles,
+ ttlCompactFiltersManager,
+ ttlTimeProvider);
+ }
+
+ /**
+ * Restores all key-groups data that is referenced by the passed state handles.
+ *
+ */
+ @Override
+ public RocksDBRestoreResult restore()
+ throws IOException, StateMigrationException, RocksDBException {
+ openDB();
+ for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
+ if (keyedStateHandle != null) {
+
+ if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
+ throw new IllegalStateException("Unexpected state handle type, " +
+ "expected: " + KeyGroupsStateHandle.class +
+ ", but found: " + keyedStateHandle.getClass());
+ }
+ this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
+ restoreKeyGroupsInStateHandle();
+ }
+ }
+ return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
+ -1, null, null);
+ }
+
+ /**
+ * Restore one key groups state handle.
+ */
+ private void restoreKeyGroupsInStateHandle()
+ throws IOException, StateMigrationException, RocksDBException {
+ try {
+ currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream();
+ cancelStreamRegistry.registerCloseable(currentStateHandleInStream);
+ currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream);
+ restoreKVStateMetaData();
+ restoreKVStateData();
+ } finally {
+ if (cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) {
+ IOUtils.closeQuietly(currentStateHandleInStream);
+ }
+ }
+ }
+
+ /**
+ * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
+ */
+ private void restoreKVStateMetaData() throws IOException, StateMigrationException, RocksDBException {
+ KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);
+
+ this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
+ SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;
+
+ List<StateMetaInfoSnapshot> restoredMetaInfos =
+ serializationProxy.getStateMetaInfoSnapshots();
+ currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());
+
+ for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
+ RocksDbKvStateInfo registeredStateCFHandle =
+ getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
+ currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
+ }
+ }
+
+ /**
+ * Restore the KV-state / ColumnFamily data for all key-groups referenced by the current state handle.
+ */
+ private void restoreKVStateData() throws IOException, RocksDBException {
+ //for all key-groups in the current state handle...
+ try (RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(db)) {
+ for (Tuple2<Integer, Long> keyGroupOffset : currentKeyGroupsStateHandle.getGroupRangeOffsets()) {
+ int keyGroup = keyGroupOffset.f0;
+
+ // Check that restored key groups all belong to the backend
+ Preconditions.checkState(keyGroupRange.contains(keyGroup),
+ "The key group must belong to the backend");
+
+ long offset = keyGroupOffset.f1;
+ //not empty key-group?
+ if (0L != offset) {
+ currentStateHandleInStream.seek(offset);
+ try (InputStream compressedKgIn = keygroupStreamCompressionDecorator.decorateWithCompression(currentStateHandleInStream)) {
+ DataInputViewStreamWrapper compressedKgInputView = new DataInputViewStreamWrapper(compressedKgIn);
+ //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
+ int kvStateId = compressedKgInputView.readShort();
+ ColumnFamilyHandle handle = currentStateHandleKVStateColumnFamilies.get(kvStateId);
+ //insert all k/v pairs into DB
+ boolean keyGroupHasMoreKeys = true;
+ while (keyGroupHasMoreKeys) {
+ byte[] key = BytePrimitiveArraySerializer.INSTANCE.deserialize(compressedKgInputView);
+ byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(compressedKgInputView);
+ if (hasMetaDataFollowsFlag(key)) {
+ //clear the signal bit in the key to make it ready for insertion again
+ clearMetaDataFollowsFlag(key);
+ writeBatchWrapper.put(handle, key, value);
+ //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
+ kvStateId = END_OF_KEY_GROUP_MARK
+ & compressedKgInputView.readShort();
+ if (END_OF_KEY_GROUP_MARK == kvStateId) {
+ keyGroupHasMoreKeys = false;
+ } else {
+ handle = currentStateHandleKVStateColumnFamilies.get(kvStateId);
+ }
+ } else {
+ writeBatchWrapper.put(handle, key, value);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java
new file mode 100644
index 0000000..24627b2
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java
@@ -0,0 +1,523 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state.restore;
+
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.contrib.streaming.state.RocksDBIncrementalCheckpointUtils;
+import org.apache.flink.contrib.streaming.state.RocksDBKeySerializationUtils;
+import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend.RocksDbKvStateInfo;
+import org.apache.flink.contrib.streaming.state.RocksDBNativeMetricOptions;
+import org.apache.flink.contrib.streaming.state.RocksDBOperationUtils;
+import org.apache.flink.contrib.streaming.state.RocksDBStateDownloader;
+import org.apache.flink.contrib.streaming.state.RocksDBWriteBatchWrapper;
+import org.apache.flink.contrib.streaming.state.RocksIteratorWrapper;
+import org.apache.flink.contrib.streaming.state.ttl.RocksDbTtlCompactFiltersManager;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.state.BackendBuildingException;
+import org.apache.flink.runtime.state.DirectoryStateHandle;
+import org.apache.flink.runtime.state.IncrementalKeyedStateHandle;
+import org.apache.flink.runtime.state.IncrementalLocalKeyedStateHandle;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.StateHandleID;
+import org.apache.flink.runtime.state.StateSerializerProvider;
+import org.apache.flink.runtime.state.StreamStateHandle;
+import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.util.IOUtils;
+
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.function.Function;
+
+import static org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.SST_FILE_SUFFIX;
+
+/**
+ * Encapsulates the process of restoring a RocksDB instance from an incremental snapshot.
+ */
+public class RocksDBIncrementalRestoreOperation<K> extends AbstractRocksDBRestoreOperation<K> {
+ private static final Logger LOG = LoggerFactory.getLogger(RocksDBIncrementalRestoreOperation.class);
+
+ private final String operatorIdentifier;
+ protected final SortedMap<Long, Set<StateHandleID>> restoredSstFiles;
+ protected long lastCompletedCheckpointId;
+ protected UUID backendUID;
+
+ public RocksDBIncrementalRestoreOperation(
+ String operatorIdentifier,
+ KeyGroupRange keyGroupRange,
+ int keyGroupPrefixBytes,
+ int numberOfTransferringThreads,
+ CloseableRegistry cancelStreamRegistry,
+ ClassLoader userCodeClassLoader,
+ Map<String, RocksDbKvStateInfo> kvStateInformation,
+ StateSerializerProvider<K> keySerializerProvider,
+ File instanceBasePath,
+ File instanceRocksDBPath,
+ DBOptions dbOptions,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+ RocksDBNativeMetricOptions nativeMetricOptions,
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> restoreStateHandles,
+ @Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
+ TtlTimeProvider ttlTimeProvider) {
+ super(keyGroupRange,
+ keyGroupPrefixBytes,
+ numberOfTransferringThreads,
+ cancelStreamRegistry,
+ userCodeClassLoader,
+ kvStateInformation,
+ keySerializerProvider,
+ instanceBasePath,
+ instanceRocksDBPath,
+ dbOptions,
+ columnFamilyOptionsFactory,
+ nativeMetricOptions,
+ metricGroup,
+ restoreStateHandles,
+ ttlCompactFiltersManager,
+ ttlTimeProvider);
+ this.operatorIdentifier = operatorIdentifier;
+ this.restoredSstFiles = new TreeMap<>();
+ this.lastCompletedCheckpointId = -1L;
+ this.backendUID = UUID.randomUUID();
+ }
+
+ /**
+ * Root method that branches for different implementations of {@link KeyedStateHandle}.
+ */
+ @Override
+ public RocksDBRestoreResult restore() throws Exception {
+
+ if (restoreStateHandles == null || restoreStateHandles.isEmpty()) {
+ return null;
+ }
+
+ final KeyedStateHandle theFirstStateHandle = restoreStateHandles.iterator().next();
+
+ boolean isRescaling = (restoreStateHandles.size() > 1 ||
+ !Objects.equals(theFirstStateHandle.getKeyGroupRange(), keyGroupRange));
+
+ if (isRescaling) {
+ restoreWithRescaling(restoreStateHandles);
+ } else {
+ restoreWithoutRescaling(theFirstStateHandle);
+ }
+ return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle,
+ nativeMetricMonitor, lastCompletedCheckpointId, backendUID, restoredSstFiles);
+ }
+
+ /**
+ * Recovery from a single remote incremental state without rescaling.
+ */
+ void restoreWithoutRescaling(KeyedStateHandle rawStateHandle) throws Exception {
+ // 1. Prepare for restore without rescaling
+ Path temporaryRestoreInstancePath = new Path(
+ instanceBasePath.getAbsolutePath(),
+ UUID.randomUUID().toString()); // used as restore source for IncrementalKeyedStateHandle
+ RocksDBIncrementalRestorePrepareResult prepareResult = prepareFiles(rawStateHandle, temporaryRestoreInstancePath);
+ Path restoreSourcePath = prepareResult.getLocalKeyedStateHandle().getDirectoryStateHandle().getDirectory();
+ if (rawStateHandle instanceof IncrementalKeyedStateHandle) {
+ backendUID = ((IncrementalKeyedStateHandle) rawStateHandle).getBackendIdentifier();
+ } else {
+ backendUID = ((IncrementalLocalKeyedStateHandle) rawStateHandle).getBackendIdentifier();
+ }
+ LOG.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.",
+ this.operatorIdentifier, this.backendUID);
+ if (!instanceRocksDBPath.mkdirs()) {
+ String errMsg = "Could not create RocksDB data directory: " + instanceBasePath.getAbsolutePath();
+ LOG.error(errMsg);
+ throw new IOException(errMsg);
+ }
+ try {
+ restoreInstanceDirectoryFromPath(restoreSourcePath, dbPath);
+ } finally {
+ cleanUpPathQuietly(restoreSourcePath);
+ }
+ // 2. Open db instance
+ openDB();
+ // 3. Use the restore sst files as the base for succeeding checkpoints
+ IncrementalLocalKeyedStateHandle restoreStateHandle = prepareResult.getLocalKeyedStateHandle();
+ restoredSstFiles.put(
+ restoreStateHandle.getCheckpointId(),
+ restoreStateHandle.getSharedStateHandleIDs());
+ lastCompletedCheckpointId = restoreStateHandle.getCheckpointId();
+ // 4. Register column family handles
+ registerCFHandles(prepareResult.getStateMetaInfoSnapshots());
+ }
+
+ private void cleanUpPathQuietly(Path path) {
+ try {
+ FileSystem fileSystem = path.getFileSystem();
+ if (fileSystem.exists(path)) {
+ fileSystem.delete(path, true);
+ }
+ } catch (IOException ex) {
+ LOG.warn("Failed to clean up path " + path, ex);
+ }
+ }
+
+ private void registerCFHandles(List<StateMetaInfoSnapshot> metaInfoSnapshots)
+ throws BackendBuildingException {
+ // Register CF handlers
+ for (int i = 0; i < metaInfoSnapshots.size(); ++i) {
+ try {
+ getOrRegisterStateColumnFamilyHandle(
+ columnFamilyHandles.get(i),
+ metaInfoSnapshots.get(i));
+ } catch (RocksDBException e) {
+ String errMsg = "Failed to register CF handle.";
+ LOG.error(errMsg, e);
+ throw new BackendBuildingException(errMsg, e);
+ }
+ }
+ }
+
+ public RocksDBIncrementalRestorePrepareResult prepareFiles(
+ KeyedStateHandle rawStateHandle,
+ Path restoreInstancePath) throws BackendBuildingException {
+ IncrementalLocalKeyedStateHandle localKeyedStateHandle;
+ List<StateMetaInfoSnapshot> stateMetaInfoSnapshots;
+ try {
+ KeyedBackendSerializationProxy<K> serializationProxy;
+ if (rawStateHandle instanceof IncrementalKeyedStateHandle) {
+ IncrementalKeyedStateHandle restoreStateHandle = (IncrementalKeyedStateHandle) rawStateHandle;
+
+ // read state data.
+ try (RocksDBStateDownloader rocksDBStateDownloader =
+ new RocksDBStateDownloader(numberOfTransferringThreads)) {
+ rocksDBStateDownloader.transferAllStateDataToDirectory(
+ restoreStateHandle,
+ restoreInstancePath,
+ cancelStreamRegistry);
+ }
+
+ serializationProxy = readMetaData(restoreStateHandle.getMetaStateHandle());
+ stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
+ columnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, true);
+
+ // since we transferred all remote state to a local directory, we can use the same code as for
+ // local recovery.
+ localKeyedStateHandle = new IncrementalLocalKeyedStateHandle(
+ restoreStateHandle.getBackendIdentifier(),
+ restoreStateHandle.getCheckpointId(),
+ new DirectoryStateHandle(restoreInstancePath),
+ restoreStateHandle.getKeyGroupRange(),
+ restoreStateHandle.getMetaStateHandle(),
+ restoreStateHandle.getSharedState().keySet());
+ } else if (rawStateHandle instanceof IncrementalLocalKeyedStateHandle) {
+ // Recovery from local incremental state.
+ localKeyedStateHandle = (IncrementalLocalKeyedStateHandle) rawStateHandle;
+ serializationProxy = readMetaData(localKeyedStateHandle.getMetaDataState());
+ stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
+ columnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, true);
+ } else {
+ throw new IllegalStateException("Unexpected state handle type, " +
+ "expected " + IncrementalKeyedStateHandle.class + " or " + IncrementalLocalKeyedStateHandle.class +
+ ", but found " + rawStateHandle.getClass());
+ }
+ } catch (Exception e) {
+ // clean up
+ cleanUpPathQuietly(restoreInstancePath);
+ // log and rethrow
+ String errMsg = "Failed to prepare files for restore from incremental state handle.";
+ LOG.error(errMsg, e);
+ throw new BackendBuildingException(errMsg, e);
+ }
+ columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size() + 1);
+ return new RocksDBIncrementalRestorePrepareResult(stateMetaInfoSnapshots, localKeyedStateHandle);
+ }
+
+ /**
+ * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary
+ * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the
+ * real restore instance and then the temporary instance is discarded.
+ */
+ void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {
+ // Init DB instance
+ initDBWithRescaling(restoreStateHandles);
+ // Do restore
+ byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
+ RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes);
+
+ byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
+ RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes);
+
+ for (KeyedStateHandle rawStateHandle : restoreStateHandles) {
+
+ if (!(rawStateHandle instanceof IncrementalKeyedStateHandle)) {
+ throw new IllegalStateException("Unexpected state handle type, " +
+ "expected " + IncrementalKeyedStateHandle.class +
+ ", but found " + rawStateHandle.getClass());
+ }
+
+ Path temporaryRestoreInstancePath = new Path(instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString());
+ try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle(
+ (IncrementalKeyedStateHandle) rawStateHandle,
+ temporaryRestoreInstancePath);
+ RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db)) {
+
+ List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors;
+ List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles;
+
+ // iterating only the requested descriptors automatically skips the default column family handle
+ for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) {
+ ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i);
+
+ ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle(
+ null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i))
+ .columnFamilyHandle;
+
+ try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) {
+
+ iterator.seek(startKeyGroupPrefixBytes);
+
+ while (iterator.isValid()) {
+
+ if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) {
+ writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value());
+ } else {
+ // Since the iterator will visit the record according to the sorted order,
+ // we can just break here.
+ break;
+ }
+
+ iterator.next();
+ }
+ } // releases native iterator resources
+ }
+ } finally {
+ cleanUpPathQuietly(temporaryRestoreInstancePath);
+ }
+ }
+ }
+
+ private void initDBWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws IOException {
+ // Prepare for restore with rescaling
+ IncrementalKeyedStateHandle initialHandle =
+ (IncrementalKeyedStateHandle) RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial(
+ restoreStateHandles, keyGroupRange);
+ Path instancePath = new Path(dbPath);
+ if (initialHandle != null) {
+ restoreStateHandles.remove(initialHandle);
+ // 1. Download target files
+ RocksDBIncrementalRestorePrepareResult prepareResult = prepareFiles(initialHandle, instancePath);
+ // 2. Open db instance
+ openDB();
+ // 3. Clip the DB instance
+ try {
+ RocksDBIncrementalCheckpointUtils.clipDBWithKeyGroupRange(
+ db,
+ columnFamilyHandles,
+ keyGroupRange,
+ initialHandle.getKeyGroupRange(),
+ keyGroupPrefixBytes);
+ } catch (RocksDBException e) {
+ String errMsg = "Failed to clip DB after initialization.";
+ LOG.error(errMsg, e);
+ throw new BackendBuildingException(errMsg, e);
+ }
+ // 4. Register column family handles
+ registerCFHandles(prepareResult.getStateMetaInfoSnapshots());
+ } else {
+ openDB();
+ }
+ }
+
+ /**
+ * Entity to hold the temporary RocksDB instance created for restore.
+ */
+ private static class RestoredDBInstance implements AutoCloseable {
+
+ @Nonnull
+ private final RocksDB db;
+
+ @Nonnull
+ private final ColumnFamilyHandle defaultColumnFamilyHandle;
+
+ @Nonnull
+ private final List<ColumnFamilyHandle> columnFamilyHandles;
+
+ @Nonnull
+ private final List<ColumnFamilyDescriptor> columnFamilyDescriptors;
+
+ @Nonnull
+ private final List<StateMetaInfoSnapshot> stateMetaInfoSnapshots;
+
+ private RestoredDBInstance(
+ @Nonnull RocksDB db,
+ @Nonnull List<ColumnFamilyHandle> columnFamilyHandles,
+ @Nonnull List<ColumnFamilyDescriptor> columnFamilyDescriptors,
+ @Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {
+ this.db = db;
+ this.defaultColumnFamilyHandle = columnFamilyHandles.remove(0);
+ this.columnFamilyHandles = columnFamilyHandles;
+ this.columnFamilyDescriptors = columnFamilyDescriptors;
+ this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
+ }
+
+ @Override
+ public void close() {
+ List<ColumnFamilyOptions> columnFamilyOptions = new ArrayList<>(columnFamilyDescriptors.size() + 1);
+ columnFamilyDescriptors.forEach((cfd) -> columnFamilyOptions.add(cfd.getOptions()));
+ RocksDBOperationUtils.addColumnFamilyOptionsToCloseLater(columnFamilyOptions, defaultColumnFamilyHandle);
+ IOUtils.closeQuietly(defaultColumnFamilyHandle);
+ IOUtils.closeAllQuietly(columnFamilyHandles);
+ IOUtils.closeQuietly(db);
+ IOUtils.closeAllQuietly(columnFamilyOptions);
+ }
+ }
+
+ private RestoredDBInstance restoreDBInstanceFromStateHandle(
+ IncrementalKeyedStateHandle restoreStateHandle,
+ Path temporaryRestoreInstancePath) throws Exception {
+
+ try (RocksDBStateDownloader rocksDBStateDownloader =
+ new RocksDBStateDownloader(numberOfTransferringThreads)) {
+ rocksDBStateDownloader.transferAllStateDataToDirectory(
+ restoreStateHandle,
+ temporaryRestoreInstancePath,
+ cancelStreamRegistry);
+ }
+
+ KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(restoreStateHandle.getMetaStateHandle());
+ // read meta data
+ List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
+
+ List<ColumnFamilyDescriptor> columnFamilyDescriptors =
+ createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, false);
+
+ List<ColumnFamilyHandle> columnFamilyHandles =
+ new ArrayList<>(stateMetaInfoSnapshots.size() + 1);
+
+ RocksDB restoreDb = RocksDBOperationUtils.openDB(
+ temporaryRestoreInstancePath.getPath(),
+ columnFamilyDescriptors,
+ columnFamilyHandles,
+ RocksDBOperationUtils.createColumnFamilyOptions(columnFamilyOptionsFactory, "default"),
+ dbOptions);
+
+ return new RestoredDBInstance(restoreDb, columnFamilyHandles, columnFamilyDescriptors, stateMetaInfoSnapshots);
+ }
+
+ /**
+ * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot.
+ */
+ private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors(
+ List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
+ boolean registerTtlCompactFilter) {
+
+ List<ColumnFamilyDescriptor> columnFamilyDescriptors =
+ new ArrayList<>(stateMetaInfoSnapshots.size());
+
+ for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) {
+ ColumnFamilyOptions options = RocksDBOperationUtils.createColumnFamilyOptions(
+ columnFamilyOptionsFactory, stateMetaInfoSnapshot.getName());
+ if (registerTtlCompactFilter) {
+ ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(ttlTimeProvider,
+ stateMetaInfoSnapshot, options);
+ }
+ ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(
+ stateMetaInfoSnapshot.getName().getBytes(ConfigConstants.DEFAULT_CHARSET),
+ options);
+
+ columnFamilyDescriptors.add(columnFamilyDescriptor);
+ }
+ return columnFamilyDescriptors;
+ }
+
+ /**
+ * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
+ * a local state.
+ */
+ public void restoreInstanceDirectoryFromPath(Path source, String instanceRocksDBPath) throws IOException {
+
+ FileSystem fileSystem = source.getFileSystem();
+
+ final FileStatus[] fileStatuses = fileSystem.listStatus(source);
+
+ if (fileStatuses == null) {
+ throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
+ }
+
+ for (FileStatus fileStatus : fileStatuses) {
+ final Path filePath = fileStatus.getPath();
+ final String fileName = filePath.getName();
+ File restoreFile = new File(source.getPath(), fileName);
+ File targetFile = new File(instanceRocksDBPath, fileName);
+ if (fileName.endsWith(SST_FILE_SUFFIX)) {
+ // hardlink'ing the immutable sst-files.
+ Files.createLink(targetFile.toPath(), restoreFile.toPath());
+ } else {
+ // true copy for all other files.
+ Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+ }
+
+ /**
+ * Reads Flink's state meta data file from the state handle.
+ */
+ private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {
+
+ FSDataInputStream inputStream = null;
+
+ try {
+ inputStream = metaStateHandle.openInputStream();
+ cancelStreamRegistry.registerCloseable(inputStream);
+ DataInputView in = new DataInputViewStreamWrapper(inputStream);
+ KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(in);
+ return serializationProxy;
+ } finally {
+ if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
+ inputStream.close();
+ }
+ }
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestorePrepareResult.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestorePrepareResult.java
new file mode 100644
index 0000000..414ccfa
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestorePrepareResult.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state.restore;
+
+import org.apache.flink.runtime.state.IncrementalLocalKeyedStateHandle;
+import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
+
+import java.util.List;
+
+/**
+ * Entity holding restore preparation result for RocksDB incremental restore.
+ */
+public class RocksDBIncrementalRestorePrepareResult {
+ private final List<StateMetaInfoSnapshot> stateMetaInfoSnapshots;
+ private final IncrementalLocalKeyedStateHandle localKeyedStateHandle;
+
+ public RocksDBIncrementalRestorePrepareResult(List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
+ IncrementalLocalKeyedStateHandle localKeyedStateHandle) {
+ this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
+ this.localKeyedStateHandle = localKeyedStateHandle;
+ }
+
+ public List<StateMetaInfoSnapshot> getStateMetaInfoSnapshots() {
+ return stateMetaInfoSnapshots;
+ }
+
+ public IncrementalLocalKeyedStateHandle getLocalKeyedStateHandle() {
+ return localKeyedStateHandle;
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBNoneRestoreOperation.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBNoneRestoreOperation.java
new file mode 100644
index 0000000..873132a
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBNoneRestoreOperation.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state.restore;
+
+import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend.RocksDbKvStateInfo;
+import org.apache.flink.contrib.streaming.state.RocksDBNativeMetricOptions;
+import org.apache.flink.contrib.streaming.state.ttl.RocksDbTtlCompactFiltersManager;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.StateSerializerProvider;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+
+import javax.annotation.Nonnull;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * Encapsulates the process of initiating a RocksDB instance without restore.
+ */
+public class RocksDBNoneRestoreOperation<K> extends AbstractRocksDBRestoreOperation<K> {
+ public RocksDBNoneRestoreOperation(
+ KeyGroupRange keyGroupRange,
+ int keyGroupPrefixBytes,
+ int numberOfTransferringThreads,
+ CloseableRegistry cancelStreamRegistry,
+ ClassLoader userCodeClassLoader,
+ Map<String, RocksDbKvStateInfo> kvStateInformation,
+ StateSerializerProvider<K> keySerializerProvider,
+ File instanceBasePath,
+ File instanceRocksDBPath,
+ DBOptions dbOptions,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+ RocksDBNativeMetricOptions nativeMetricOptions,
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> restoreStateHandles,
+ @Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
+ TtlTimeProvider ttlTimeProvider
+ ) {
+ super(keyGroupRange,
+ keyGroupPrefixBytes,
+ numberOfTransferringThreads,
+ cancelStreamRegistry,
+ userCodeClassLoader,
+ kvStateInformation,
+ keySerializerProvider,
+ instanceBasePath,
+ instanceRocksDBPath,
+ dbOptions,
+ columnFamilyOptionsFactory,
+ nativeMetricOptions,
+ metricGroup,
+ restoreStateHandles,
+ ttlCompactFiltersManager,
+ ttlTimeProvider);
+ }
+
+ @Override
+ public RocksDBRestoreResult restore() throws Exception {
+ openDB();
+ return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
+ -1, null, null);
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBRestoreOperation.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBRestoreOperation.java
new file mode 100644
index 0000000..ff70199
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBRestoreOperation.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state.restore;
+
+/**
+ * Interface for RocksDB restore.
+ */
+public interface RocksDBRestoreOperation {
+ /**
+ * Restores state that was previously snapshot-ed from the provided state handles.
+ */
+ RocksDBRestoreResult restore() throws Exception;
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBRestoreResult.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBRestoreResult.java
new file mode 100644
index 0000000..d53eb50
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBRestoreResult.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state.restore;
+
+import org.apache.flink.contrib.streaming.state.RocksDBNativeMetricMonitor;
+import org.apache.flink.runtime.state.StateHandleID;
+
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.RocksDB;
+
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.UUID;
+
+/**
+ * Entity holding result of RocksDB instance restore.
+ */
+public class RocksDBRestoreResult {
+ private final RocksDB db;
+ private final ColumnFamilyHandle defaultColumnFamilyHandle;
+ private final RocksDBNativeMetricMonitor nativeMetricMonitor;
+
+ // fields only for incremental restore
+ private final long lastCompletedCheckpointId;
+ private final UUID backendUID;
+ private final SortedMap<Long, Set<StateHandleID>> restoredSstFiles;
+
+ public RocksDBRestoreResult(
+ RocksDB db,
+ ColumnFamilyHandle defaultColumnFamilyHandle,
+ RocksDBNativeMetricMonitor nativeMetricMonitor,
+ long lastCompletedCheckpointId,
+ UUID backendUID,
+ SortedMap<Long, Set<StateHandleID>> restoredSstFiles) {
+ this.db = db;
+ this.defaultColumnFamilyHandle = defaultColumnFamilyHandle;
+ this.nativeMetricMonitor = nativeMetricMonitor;
+ this.lastCompletedCheckpointId = lastCompletedCheckpointId;
+ this.backendUID = backendUID;
+ this.restoredSstFiles = restoredSstFiles;
+ }
+
+ public RocksDB getDb() {
+ return db;
+ }
+
+ public long getLastCompletedCheckpointId() {
+ return lastCompletedCheckpointId;
+ }
+
+ public UUID getBackendUID() {
+ return backendUID;
+ }
+
+ public SortedMap<Long, Set<StateHandleID>> getRestoredSstFiles() {
+ return restoredSstFiles;
+ }
+
+ public ColumnFamilyHandle getDefaultColumnFamilyHandle() {
+ return defaultColumnFamilyHandle;
+ }
+
+ public RocksDBNativeMetricMonitor getNativeMetricMonitor() {
+ return nativeMetricMonitor;
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksDBSnapshotStrategyBase.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksDBSnapshotStrategyBase.java
index 8937d3e..d517a74 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksDBSnapshotStrategyBase.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksDBSnapshotStrategyBase.java
@@ -56,7 +56,7 @@ public abstract class RocksDBSnapshotStrategyBase<K>
/** RocksDB instance from the backend. */
@Nonnull
- protected final RocksDB db;
+ protected RocksDB db;
/** Resource guard for the RocksDB instance. */
@Nonnull
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java
index 354f735..d0b69fa 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java
@@ -399,9 +399,6 @@ public class RocksDBAsyncSnapshotTest extends TestLogger {
null);
try {
-
- keyedStateBackend.restore(null);
-
// register a state so that the state backend has to checkpoint something
keyedStateBackend.getPartitionedState(
"namespace",
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBNativeMetricMonitorTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBNativeMetricMonitorTest.java
index 039fba9..92f1b60 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBNativeMetricMonitorTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBNativeMetricMonitorTest.java
@@ -71,9 +71,9 @@ public class RocksDBNativeMetricMonitorTest {
options.enableSizeAllMemTables();
RocksDBNativeMetricMonitor monitor = new RocksDBNativeMetricMonitor(
- localRocksDBResource.getRocksDB(),
options,
- group
+ group,
+ localRocksDBResource.getRocksDB()
);
ColumnFamilyHandle handle = localRocksDBResource.createNewColumnFamily(COLUMN_FAMILY_NAME);
@@ -116,9 +116,9 @@ public class RocksDBNativeMetricMonitorTest {
options.enableSizeAllMemTables();
RocksDBNativeMetricMonitor monitor = new RocksDBNativeMetricMonitor(
- localRocksDBResource.getRocksDB(),
options,
- group
+ group,
+ localRocksDBResource.getRocksDB()
);
ColumnFamilyHandle handle = rocksDBResource.createNewColumnFamily(COLUMN_FAMILY_NAME);
@@ -146,9 +146,9 @@ public class RocksDBNativeMetricMonitorTest {
options.enableSizeAllMemTables();
RocksDBNativeMetricMonitor monitor = new RocksDBNativeMetricMonitor(
- rocksDBResource.getRocksDB(),
options,
- group
+ group,
+ rocksDBResource.getRocksDB()
);
ColumnFamilyHandle handle = rocksDBResource.createNewColumnFamily(COLUMN_FAMILY_NAME);
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBRocksStateKeysIteratorTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBRocksStateKeysIteratorTest.java
index abf32b6..87e06b6 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBRocksStateKeysIteratorTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBRocksStateKeysIteratorTest.java
@@ -93,7 +93,6 @@ public class RocksDBRocksStateKeysIteratorTest {
mock(TaskKvStateRegistry.class));
try {
- keyedStateBackend.restore(null);
ValueState<String> testState = keyedStateBackend.getPartitionedState(
namespace,
namespaceSerializer,
@@ -119,7 +118,7 @@ public class RocksDBRocksStateKeysIteratorTest {
ColumnFamilyHandle handle = keyedStateBackend.getColumnFamilyHandle(testStateName);
try (
- RocksIteratorWrapper iterator = RocksDBKeyedStateBackend.getRocksIterator(keyedStateBackend.db, handle);
+ RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(keyedStateBackend.db, handle);
RocksStateKeysIterator<K> iteratorWrapper =
new RocksStateKeysIterator<>(
iterator,
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendConfigTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendConfigTest.java
index 57aa063..7329646 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendConfigTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendConfigTest.java
@@ -50,7 +50,6 @@ import org.rocksdb.CompactionStyle;
import org.rocksdb.DBOptions;
import java.io.File;
-import java.util.Collections;
import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -149,7 +148,6 @@ public class RocksDBStateBackendConfigTest {
RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(tempFolder.newFolder().toURI().toString());
RocksDBKeyedStateBackend<Integer> keyedBackend = createKeyedStateBackend(rocksDbBackend, env);
- keyedBackend.restore(Collections.emptyList());
Assert.assertEquals(HeapPriorityQueueSetFactory.class, keyedBackend.getPriorityQueueFactory().getClass());
keyedBackend.dispose();
@@ -160,9 +158,8 @@ public class RocksDBStateBackendConfigTest {
rocksDbBackend = rocksDbBackend.configure(conf, Thread.currentThread().getContextClassLoader());
keyedBackend = createKeyedStateBackend(rocksDbBackend, env);
- keyedBackend.restore(Collections.emptyList());
Assert.assertEquals(
- RocksDBKeyedStateBackend.RocksDBPriorityQueueSetFactory.class,
+ RocksDBPriorityQueueSetFactory.class,
keyedBackend.getPriorityQueueFactory().getClass());
keyedBackend.dispose();
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
index 0b7a578..b681a87 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
@@ -19,7 +19,6 @@
package org.apache.flink.contrib.streaming.state;
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeutils.base.IntSerializer;
@@ -35,21 +34,21 @@ import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.IncrementalKeyedStateHandle;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.LocalRecoveryConfig;
import org.apache.flink.runtime.state.SharedStateRegistry;
import org.apache.flink.runtime.state.SnapshotResult;
import org.apache.flink.runtime.state.StateBackendTestBase;
import org.apache.flink.runtime.state.StateHandleID;
import org.apache.flink.runtime.state.StreamStateHandle;
-import org.apache.flink.runtime.state.TestLocalRecoveryConfig;
import org.apache.flink.runtime.state.VoidNamespace;
import org.apache.flink.runtime.state.VoidNamespaceSerializer;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.apache.flink.runtime.util.BlockerCheckpointStreamFactory;
import org.apache.flink.runtime.util.BlockingCheckpointOutputStream;
+import org.apache.flink.util.IOUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.junit.After;
import org.junit.Rule;
@@ -75,6 +74,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -83,6 +83,7 @@ import java.util.Queue;
import java.util.concurrent.RunnableFuture;
import static junit.framework.TestCase.assertNotNull;
+import static org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.DB_INSTANCE_DIR_STRING;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -121,6 +122,23 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
// Store it because we need it for the cleanup test.
private String dbPath;
+ private RocksDB db = null;
+ private File instanceBasePath = null;
+ private ColumnFamilyHandle defaultCFHandle = null;
+ private ColumnFamilyOptions columnOptions = null;
+ private DBOptions dbOptions = null;
+
+ public void prepareRocksDB() throws Exception {
+ instanceBasePath = tempFolder.newFolder();
+ instanceBasePath.mkdirs();
+ String dbPath = new File(instanceBasePath, DB_INSTANCE_DIR_STRING).getAbsolutePath();
+ columnOptions = PredefinedOptions.DEFAULT.createColumnOptions();
+ dbOptions = PredefinedOptions.DEFAULT.createDBOptions().setCreateIfMissing(true);
+ ArrayList<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(1);
+ db = RocksDBOperationUtils.openDB(dbPath, Collections.emptyList(),
+ columnFamilyHandles, columnOptions, dbOptions);
+ defaultCFHandle = columnFamilyHandles.remove(0);
+ }
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
@@ -148,6 +166,10 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
IOUtils.closeQuietly(keyedStateBackend);
keyedStateBackend.dispose();
}
+ IOUtils.closeQuietly(defaultCFHandle);
+ IOUtils.closeQuietly(db);
+ IOUtils.closeQuietly(columnOptions);
+ IOUtils.closeQuietly(dbOptions);
if (allCreatedCloseables != null) {
for (RocksObject rocksCloseable : allCreatedCloseables) {
@@ -155,6 +177,11 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
}
allCreatedCloseables = null;
}
+ try {
+ org.apache.flink.util.FileUtils.deleteDirectory(instanceBasePath);
+ } catch (Exception ex) {
+ // ignored
+ }
}
public void setupRocksKeyedStateBackend() throws Exception {
@@ -166,19 +193,29 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
testStreamFactory.setWaiterLatch(waiter);
testStreamFactory.setAfterNumberInvocations(10);
- RocksDBStateBackend backend = getStateBackend();
+ prepareRocksDB();
Environment env = new DummyEnvironment("TestTask", 1, 0);
- keyedStateBackend = (RocksDBKeyedStateBackend<Integer>) backend.createKeyedStateBackend(
- env,
- new JobID(),
- "Test",
- IntSerializer.INSTANCE,
- 2,
- new KeyGroupRange(0, 1),
- mock(TaskKvStateRegistry.class));
-
- keyedStateBackend.restore(null);
+ keyedStateBackend = new RocksDBKeyedStateBackendBuilder<>(
+ "Test",
+ Thread.currentThread().getContextClassLoader(),
+ instanceBasePath,
+ dbOptions,
+ stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
+ mock(TaskKvStateRegistry.class),
+ IntSerializer.INSTANCE,
+ 2,
+ new KeyGroupRange(0, 1),
+ env.getExecutionConfig(),
+ env.getTaskStateManager().createLocalRecoveryConfig(),
+ RocksDBStateBackend.PriorityQueueStateType.ROCKSDB,
+ TtlTimeProvider.DEFAULT,
+ new UnregisteredMetricsGroup(),
+ Collections.emptyList(),
+ RocksDBStateBackend.getCompressionDecorator(env.getExecutionConfig()),
+ spy(db),
+ defaultCFHandle
+ ).build();
testState1 = keyedStateBackend.getPartitionedState(
VoidNamespace.INSTANCE,
@@ -192,9 +229,6 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
allCreatedCloseables = new ArrayList<>();
- keyedStateBackend.db = spy(keyedStateBackend.db);
- keyedStateBackend.initializeSnapshotStrategy(null);
-
doAnswer(new Answer<Object>() {
@Override
@@ -234,11 +268,12 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
@Test
public void testCorrectMergeOperatorSet() throws Exception {
-
+ prepareRocksDB();
final ColumnFamilyOptions columnFamilyOptions = spy(new ColumnFamilyOptions());
RocksDBKeyedStateBackend<Integer> test = null;
try (DBOptions options = new DBOptions().setCreateIfMissing(true)) {
- test = new RocksDBKeyedStateBackend<>(
+ ExecutionConfig executionConfig = new ExecutionConfig();
+ test = new RocksDBKeyedStateBackendBuilder<>(
"test",
Thread.currentThread().getContextClassLoader(),
tempFolder.newFolder(),
@@ -248,21 +283,24 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
IntSerializer.INSTANCE,
1,
new KeyGroupRange(0, 0),
- new ExecutionConfig(),
- enableIncrementalCheckpointing,
- 1,
- TestLocalRecoveryConfig.disabled(),
+ executionConfig,
+ mock(LocalRecoveryConfig.class),
RocksDBStateBackend.PriorityQueueStateType.HEAP,
TtlTimeProvider.DEFAULT,
- false,
- new RocksDBNativeMetricOptions(),
- new UnregisteredMetricsGroup());
- test.restore(null);
- ValueStateDescriptor<String> stubState =
- new ValueStateDescriptor<>("StubState", StringSerializer.INSTANCE);
- test.createInternalState(StringSerializer.INSTANCE, stubState);
-
- // 1 time for default CF and 1 for StubState
+ new UnregisteredMetricsGroup(),
+ Collections.emptyList(),
+ RocksDBStateBackend.getCompressionDecorator(executionConfig),
+ db,
+ defaultCFHandle
+ ).build();
+ ValueStateDescriptor<String> stubState1 =
+ new ValueStateDescriptor<>("StubState-1", StringSerializer.INSTANCE);
+ test.createInternalState(StringSerializer.INSTANCE, stubState1);
+ ValueStateDescriptor<String> stubState2 =
+ new ValueStateDescriptor<>("StubState-2", StringSerializer.INSTANCE);
+ test.createInternalState(StringSerializer.INSTANCE, stubState2);
+
+ // The default CF is pre-created so sum up to 2 times (once for each stub state)
verify(columnFamilyOptions, Mockito.times(2))
.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);
} finally {
@@ -299,7 +337,7 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
this.keyedStateBackend.dispose();
verify(spyDB, times(1)).close();
- assertEquals(null, keyedStateBackend.db);
+ assertEquals(true, keyedStateBackend.isDisposed());
//Ensure every RocksObjects was closed exactly once
for (RocksObject rocksCloseable : allCreatedCloseables) {
@@ -554,7 +592,7 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
keyedStateBackend.dispose();
verify(spyDB, times(1)).close();
- assertEquals(null, keyedStateBackend.db);
+ assertEquals(true, keyedStateBackend.isDisposed());
}
private static class AcceptAllFilter implements IOFileFilter {
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksKeyGroupsRocksSingleStateIteratorTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksKeyGroupsRocksSingleStateIteratorTest.java
index e1240a8..4447f08 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksKeyGroupsRocksSingleStateIteratorTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksKeyGroupsRocksSingleStateIteratorTest.java
@@ -108,7 +108,7 @@ public class RocksKeyGroupsRocksSingleStateIteratorTest {
int id = 0;
for (Tuple2<ColumnFamilyHandle, Integer> columnFamilyHandle : columnFamilyHandlesWithKeyCount) {
- rocksIteratorsWithKVStateId.add(new Tuple2<>(RocksDBKeyedStateBackend.getRocksIterator(rocksDB, columnFamilyHandle.f0), id));
+ rocksIteratorsWithKVStateId.add(new Tuple2<>(RocksDBOperationUtils.getRocksIterator(rocksDB, columnFamilyHandle.f0), id));
++id;
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
index 8724615..96003d6 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
@@ -19,6 +19,7 @@
package org.apache.flink.contrib.streaming.state.benchmark;
import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend;
+import org.apache.flink.contrib.streaming.state.RocksDBOperationUtils;
import org.apache.flink.contrib.streaming.state.RocksIteratorWrapper;
import org.apache.flink.core.memory.MemoryUtils;
import org.apache.flink.testutils.junit.RetryOnFailure;
@@ -166,7 +167,7 @@ public class RocksDBPerformanceTest extends TestLogger {
int pos = 0;
- try (final RocksIteratorWrapper iterator = RocksDBKeyedStateBackend.getRocksIterator(rocksDB)) {
+ try (final RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(rocksDB)) {
// seek to start
unsafe.putInt(keyTemplate, offset, 0);
iterator.seek(keyTemplate);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java
index 29746dc..81b2bb7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java
@@ -25,7 +25,7 @@ import org.apache.flink.util.Disposable;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkException;
import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.function.SupplierWithException;
+import org.apache.flink.util.function.FunctionWithException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,7 +57,7 @@ public class BackendRestorerProcedure<
private static final Logger LOG = LoggerFactory.getLogger(BackendRestorerProcedure.class);
/** Factory for new, fresh backends without state. */
- private final SupplierWithException<T, Exception> instanceSupplier;
+ private final FunctionWithException<Collection<S>, T, Exception> instanceSupplier;
/** This registry is used so that recovery can participate in the task lifecycle, i.e. can be canceled. */
private final CloseableRegistry backendCloseableRegistry;
@@ -72,7 +72,7 @@ public class BackendRestorerProcedure<
* @param backendCloseableRegistry registry to allow participation in task lifecycle, e.g. react to cancel.
*/
public BackendRestorerProcedure(
- @Nonnull SupplierWithException<T, Exception> instanceSupplier,
+ @Nonnull FunctionWithException<Collection<S>, T, Exception> instanceSupplier,
@Nonnull CloseableRegistry backendCloseableRegistry,
@Nonnull String logDescription) {
@@ -141,13 +141,14 @@ public class BackendRestorerProcedure<
private T attemptCreateAndRestore(Collection<S> restoreState) throws Exception {
// create a new, empty backend.
- final T backendInstance = instanceSupplier.get();
+ final T backendInstance = instanceSupplier.apply(restoreState);
try {
// register the backend with the registry to participate in task lifecycle w.r.t. cancellation.
backendCloseableRegistry.registerCloseable(backendInstance);
// attempt to restore from snapshot (or null if no state was checkpointed).
+ // TODO we could remove this invocation when moving all backend's restore into builder
backendInstance.restore(restoreState);
return backendInstance;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
index 42abb51..0ed2330 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
@@ -238,7 +238,7 @@ public class StreamTaskStateInitializerImpl implements StreamTaskStateInitialize
BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> backendRestorer =
new BackendRestorerProcedure<>(
- () -> stateBackend.createOperatorStateBackend(environment, operatorIdentifierText),
+ (stateHandles) -> stateBackend.createOperatorStateBackend(environment, operatorIdentifierText),
backendCloseableRegistry,
logDescription);
@@ -268,7 +268,7 @@ public class StreamTaskStateInitializerImpl implements StreamTaskStateInitialize
BackendRestorerProcedure<AbstractKeyedStateBackend<K>, KeyedStateHandle> backendRestorer =
new BackendRestorerProcedure<>(
- () -> stateBackend.createKeyedStateBackend(
+ (stateHandles) -> stateBackend.createKeyedStateBackend(
environment,
environment.getJobID(),
operatorIdentifierText,
@@ -277,7 +277,8 @@ public class StreamTaskStateInitializerImpl implements StreamTaskStateInitialize
keyGroupRange,
environment.getTaskKvStateRegistry(),
TtlTimeProvider.DEFAULT,
- metricGroup),
+ metricGroup,
+ stateHandles),
backendCloseableRegistry,
logDescription);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java
index 0f15d11..876d35e 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java
@@ -34,12 +34,13 @@ import org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory;
import org.apache.flink.runtime.util.BlockingFSDataInputStream;
import org.apache.flink.util.FlinkException;
import org.apache.flink.util.TestLogger;
-import org.apache.flink.util.function.SupplierWithException;
+import org.apache.flink.util.function.FunctionWithException;
import org.junit.Assert;
import org.junit.Test;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -57,8 +58,8 @@ import static org.powermock.api.mockito.PowerMockito.when;
*/
public class BackendRestorerProcedureTest extends TestLogger {
- private final SupplierWithException<OperatorStateBackend, Exception> backendSupplier =
- () -> new DefaultOperatorStateBackend(
+ private final FunctionWithException<Collection<OperatorStateHandle>, OperatorStateBackend, Exception> backendSupplier =
+ (stateHandles) -> new DefaultOperatorStateBackend(
getClass().getClassLoader(),
new ExecutionConfig(),
true);
@@ -74,7 +75,7 @@ public class BackendRestorerProcedureTest extends TestLogger {
CheckpointStreamFactory checkpointStreamFactory = new MemCheckpointStreamFactory(1024);
ListStateDescriptor<Integer> stateDescriptor = new ListStateDescriptor<>("test-state", Integer.class);
- OperatorStateBackend originalBackend = backendSupplier.get();
+ OperatorStateBackend originalBackend = backendSupplier.apply(null);
SnapshotResult<OperatorStateHandle> snapshotResult;
try {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImplTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImplTest.java
index 3dafa02..c04a17f 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImplTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImplTest.java
@@ -38,6 +38,7 @@ import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.KeyGroupStatePartitionStreamProvider;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.OperatorStateHandle;
import org.apache.flink.runtime.state.OperatorStreamStateHandle;
@@ -59,6 +60,7 @@ import org.junit.Test;
import java.io.Closeable;
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import java.util.Random;
@@ -144,7 +146,7 @@ public class StreamTaskStateInitializerImplTest {
int numberOfKeyGroups, KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry,
TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) throws Exception {
+ MetricGroup metricGroup, Collection<KeyedStateHandle> stateHandles) throws Exception {
return mock(AbstractKeyedStateBackend.class);
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTerminationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTerminationTest.java
index d6266d3..c4ca305 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTerminationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTerminationTest.java
@@ -57,6 +57,7 @@ import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.CheckpointStreamFactory;
import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.OperatorStateHandle;
import org.apache.flink.runtime.state.SnapshotResult;
@@ -82,6 +83,7 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
@@ -267,7 +269,8 @@ public class StreamTaskTerminationTest extends TestLogger {
KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry,
TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) {
+ MetricGroup metricGroup,
+ Collection<KeyedStateHandle> stateHandles) {
return null;
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/TestSpyWrapperStateBackend.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/TestSpyWrapperStateBackend.java
index 15f1ddd..9207594 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/TestSpyWrapperStateBackend.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/TestSpyWrapperStateBackend.java
@@ -28,11 +28,13 @@ import org.apache.flink.runtime.state.AbstractStateBackend;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.apache.flink.util.Preconditions;
import java.io.IOException;
+import java.util.Collection;
import static org.powermock.api.mockito.PowerMockito.spy;
@@ -57,7 +59,7 @@ public class TestSpyWrapperStateBackend extends AbstractStateBackend {
KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry,
TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) throws IOException {
+ MetricGroup metricGroup, Collection<KeyedStateHandle> stateHandles) throws IOException {
return spy(delegate.createKeyedStateBackend(
env,
jobID,
@@ -67,7 +69,8 @@ public class TestSpyWrapperStateBackend extends AbstractStateBackend {
keyGroupRange,
kvStateRegistry,
ttlTimeProvider,
- metricGroup));
+ metricGroup,
+ null));
}
@Override
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java
index d1543b5..a87998f 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java
@@ -33,6 +33,7 @@ import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.StateBackend;
import org.apache.flink.runtime.state.memory.MemoryBackendCheckpointStorage;
@@ -44,6 +45,7 @@ import org.apache.flink.util.ExceptionUtils;
import org.junit.Test;
import java.io.IOException;
+import java.util.Collection;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -115,7 +117,8 @@ public class StateBackendITCase extends AbstractTestBase {
KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry,
TtlTimeProvider ttlTimeProvider,
- MetricGroup metricGroup) throws IOException {
+ MetricGroup metricGroup,
+ Collection<KeyedStateHandle> stateHandles) throws IOException {
throw new SuccessException();
}