You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ha...@apache.org on 2024/04/26 01:48:56 UTC

(flink) 06/07: [FLINK-35047][state] Introduce ForStStateBackend and ForStKeyedStateBackend

This is an automated email from the ASF dual-hosted git repository.

hangxiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 417484e6029c879a38b698434b9a2f8ba94413fb
Author: Hangxiang Yu <ma...@gmail.com>
AuthorDate: Tue Apr 16 23:34:36 2024 +0800

    [FLINK-35047][state] Introduce ForStStateBackend and ForStKeyedStateBackend
---
 .../flink/state/forst/ForStKeyedStateBackend.java  | 134 ++++
 .../state/forst/ForStKeyedStateBackendBuilder.java | 187 +++++
 .../flink/state/forst/ForStStateBackend.java       | 655 ++++++++++++++++++
 .../state/forst/ForStStateBackendFactory.java      |  34 +
 .../apache/flink/state/forst/ForStInitITCase.java  |  64 ++
 .../state/forst/ForStMultiClassLoaderTest.java     |  88 +++
 .../state/forst/ForStStateBackendConfigTest.java   | 767 +++++++++++++++++++++
 .../state/forst/ForStStateBackendFactoryTest.java  | 132 ++++
 .../apache/flink/state/forst/ForStTestUtils.java   |  54 ++
 9 files changed, 2115 insertions(+)

diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackend.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackend.java
new file mode 100644
index 00000000000..8c7dfd0ba0f
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackend.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.Preconditions;
+
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.RocksDB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * A KeyedStateBackend that stores its state in {@code ForSt}. This state backend can store very
+ * large state that exceeds memory even disk to remote storage. TODO: Support to implement the new
+ * interface of KeyedStateBackend
+ */
+public class ForStKeyedStateBackend<K> {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ForStKeyedStateBackend.class);
+
+    /** The key serializer. */
+    protected final TypeSerializer<K> keySerializer;
+
+    /** The container of ForSt options. */
+    private final ForStResourceContainer optionsContainer;
+
+    /** Path where this configured instance stores its local data directory. */
+    private final File localBasePath;
+
+    /**
+     * We are not using the default column family for Flink state ops, but we still need to remember
+     * this handle so that we can close it properly when the backend is closed. Note that the one
+     * returned by {@link RocksDB#open(String)} is different from that by {@link
+     * RocksDB#getDefaultColumnFamily()}, probably it's a bug of RocksDB java API.
+     */
+    private final ColumnFamilyHandle defaultColumnFamily;
+
+    /** The native metrics monitor. */
+    private final ForStNativeMetricMonitor nativeMetricMonitor;
+
+    /**
+     * Our ForSt database. The different k/v states that we have don't each have their own ForSt
+     * instance. They all write to this instance but to their own column family.
+     */
+    protected final RocksDB db;
+
+    // mark whether this backend is already disposed and prevent duplicate disposing
+    private boolean disposed = false;
+
+    public ForStKeyedStateBackend(
+            File localBasePath,
+            ForStResourceContainer optionsContainer,
+            TypeSerializer<K> keySerializer,
+            RocksDB db,
+            ColumnFamilyHandle defaultColumnFamilyHandle,
+            ForStNativeMetricMonitor nativeMetricMonitor) {
+
+        this.localBasePath = Preconditions.checkNotNull(localBasePath);
+        this.optionsContainer = Preconditions.checkNotNull(optionsContainer);
+        this.keySerializer = keySerializer;
+        this.db = db;
+        this.defaultColumnFamily = defaultColumnFamilyHandle;
+        this.nativeMetricMonitor = nativeMetricMonitor;
+    }
+
+    /** Should only be called by one thread, and only after all accesses to the DB happened. */
+    public void dispose() {
+        if (this.disposed) {
+            return;
+        }
+
+        // IMPORTANT: null reference to signal potential async checkpoint workers that the db was
+        // disposed, as
+        // working on the disposed object results in SEGFAULTS.
+        if (db != null) {
+
+            // Metric collection occurs on a background thread. When this method returns
+            // it is guaranteed that thr ForSt reference has been invalidated
+            // and no more metric collection will be attempted against the database.
+            if (nativeMetricMonitor != null) {
+                nativeMetricMonitor.close();
+            }
+
+            IOUtils.closeQuietly(defaultColumnFamily);
+
+            // ... and finally close the DB instance ...
+            IOUtils.closeQuietly(db);
+
+            IOUtils.closeQuietly(optionsContainer);
+
+            cleanLocalBasePath();
+        }
+        this.disposed = true;
+    }
+
+    private void cleanLocalBasePath() {
+        LOG.info(
+                "Closed ForSt State Backend. Cleaning up ForSt local working directory {}.",
+                localBasePath);
+
+        try {
+            FileUtils.deleteDirectory(localBasePath);
+        } catch (IOException ex) {
+            LOG.warn("Could not delete ForSt local working directory: {}", localBasePath, ex);
+        }
+    }
+
+    @VisibleForTesting
+    File getInstanceBasePath() {
+        return localBasePath;
+    }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackendBuilder.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackendBuilder.java
new file mode 100644
index 00000000000..1ff68f413a4
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackendBuilder.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.state.BackendBuildingException;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.StateBackendBuilder;
+import org.apache.flink.runtime.state.StateSerializerProvider;
+import org.apache.flink.state.forst.restore.ForStNoneRestoreOperation;
+import org.apache.flink.state.forst.restore.ForStRestoreOperation;
+import org.apache.flink.state.forst.restore.ForStRestoreResult;
+import org.apache.flink.util.CollectionUtil;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.Preconditions;
+
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.function.Function;
+
+/**
+ * Builder class for {@link ForStKeyedStateBackend} which handles all necessary initializations and
+ * cleanups.
+ *
+ * @param <K> The data type that the key serializer serializes.
+ */
+public class ForStKeyedStateBackendBuilder<K>
+        implements StateBackendBuilder<ForStKeyedStateBackend<K>, BackendBuildingException> {
+
+    protected final Logger logger = LoggerFactory.getLogger(getClass());
+
+    private final StateSerializerProvider<K> keySerializerProvider;
+    private final Collection<KeyedStateHandle> restoreStateHandles;
+
+    /** Factory function to create column family options from state name. */
+    private final Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory;
+
+    /** The container of ForSt option factory and predefined options. */
+    private final ForStResourceContainer optionsContainer;
+
+    /** Path where this configured instance stores its data directory. */
+    private final File localBasePath;
+
+    /** Path where this configured instance stores its ForSt database. */
+    private final File localForStPath;
+
+    private final MetricGroup metricGroup;
+
+    /** ForSt property-based and statistics-based native metrics options. */
+    private ForStNativeMetricOptions nativeMetricOptions;
+
+    public ForStKeyedStateBackendBuilder(
+            ForStResourceContainer optionsContainer,
+            Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+            TypeSerializer<K> keySerializer,
+            MetricGroup metricGroup,
+            @Nonnull Collection<KeyedStateHandle> stateHandles) {
+        this.optionsContainer = optionsContainer;
+        this.localBasePath = optionsContainer.getLocalBasePath();
+        this.localForStPath = optionsContainer.getLocalForStPath();
+        this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);
+        this.keySerializerProvider =
+                StateSerializerProvider.fromNewRegisteredSerializer(keySerializer);
+        this.metricGroup = metricGroup;
+        this.restoreStateHandles = stateHandles;
+        this.nativeMetricOptions = new ForStNativeMetricOptions();
+    }
+
+    ForStKeyedStateBackendBuilder<K> setNativeMetricOptions(
+            ForStNativeMetricOptions nativeMetricOptions) {
+        this.nativeMetricOptions = nativeMetricOptions;
+        return this;
+    }
+
+    @Override
+    public ForStKeyedStateBackend<K> build() throws BackendBuildingException {
+        ColumnFamilyHandle defaultColumnFamilyHandle = null;
+        ForStNativeMetricMonitor nativeMetricMonitor = null;
+        RocksDB db = null;
+        ForStRestoreOperation restoreOperation = null;
+
+        try {
+            prepareLocalDirectories();
+            restoreOperation = getForStRestoreOperation();
+            ForStRestoreResult restoreResult = restoreOperation.restore();
+            db = restoreResult.getDb();
+            defaultColumnFamilyHandle = restoreResult.getDefaultColumnFamilyHandle();
+            nativeMetricMonitor = restoreResult.getNativeMetricMonitor();
+
+            // TODO: Support to init snapshot strategy
+
+        } catch (Throwable e) {
+            // Do clean up
+            IOUtils.closeQuietly(defaultColumnFamilyHandle);
+            IOUtils.closeQuietly(nativeMetricMonitor);
+            IOUtils.closeQuietly(db);
+            // it's possible that db has been initialized but later restore steps failed
+            IOUtils.closeQuietly(restoreOperation);
+            IOUtils.closeQuietly(optionsContainer);
+            try {
+                FileUtils.deleteDirectory(localBasePath);
+            } catch (Exception ex) {
+                logger.warn("Failed to delete local base path for ForSt: " + localBasePath, ex);
+            }
+            // Log and rethrow
+            if (e instanceof BackendBuildingException) {
+                throw (BackendBuildingException) e;
+            } else {
+                String errMsg = "Caught unexpected exception.";
+                logger.error(errMsg, e);
+                throw new BackendBuildingException(errMsg, e);
+            }
+        }
+        logger.info(
+                "Finished building ForSt keyed state-backend at local base path: {}.",
+                localBasePath);
+        return new ForStKeyedStateBackend<>(
+                this.localBasePath,
+                this.optionsContainer,
+                this.keySerializerProvider.currentSchemaSerializer(),
+                db,
+                defaultColumnFamilyHandle,
+                nativeMetricMonitor);
+    }
+
+    private ForStRestoreOperation getForStRestoreOperation() {
+        DBOptions dbOptions = optionsContainer.getDbOptions();
+        if (CollectionUtil.isEmptyOrAllElementsNull(restoreStateHandles)) {
+            return new ForStNoneRestoreOperation(
+                    localForStPath,
+                    dbOptions,
+                    columnFamilyOptionsFactory,
+                    nativeMetricOptions,
+                    metricGroup);
+        }
+        // TODO: Support Restoring
+        throw new UnsupportedOperationException("Not support restoring yet for ForStStateBackend");
+    }
+
+    private void prepareLocalDirectories() throws IOException {
+        checkAndCreateDirectory(localBasePath);
+        if (localForStPath.exists()) {
+            // Clear the base directory when the backend is created
+            // in case something crashed and the backend never reached dispose()
+            FileUtils.deleteDirectory(localBasePath);
+        }
+    }
+
+    private static void checkAndCreateDirectory(File directory) throws IOException {
+        if (directory.exists()) {
+            if (!directory.isDirectory()) {
+                throw new IOException("Not a directory: " + directory);
+            }
+        } else if (!directory.mkdirs()) {
+            throw new IOException(
+                    String.format("Could not create ForSt data directory at %s.", directory));
+        }
+    }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackend.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackend.java
new file mode 100644
index 00000000000..be21b54db18
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackend.java
@@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.memory.OpaqueMemoryResource;
+import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
+import org.apache.flink.runtime.state.AbstractManagedMemoryStateBackend;
+import org.apache.flink.runtime.state.ConfigurableStateBackend;
+import org.apache.flink.runtime.state.DefaultOperatorStateBackendBuilder;
+import org.apache.flink.runtime.state.OperatorStateBackend;
+import org.apache.flink.state.forst.ForStMemoryControllerUtils.ForStMemoryFactory;
+import org.apache.flink.util.AbstractID;
+import org.apache.flink.util.DynamicCodeLoadingException;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkRuntimeException;
+
+import org.rocksdb.NativeLibraryLoader;
+import org.rocksdb.RocksDB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.Random;
+import java.util.UUID;
+import java.util.function.Supplier;
+
+/**
+ * A {@link org.apache.flink.runtime.state.StateBackend} that stores its state in a ForSt instance.
+ * This state backend can store very large state that exceeds memory even disk and spills to remote
+ * storage.
+ *
+ * <p>The behavior of the ForSt instances can be parametrized by setting ForSt Options using the
+ * methods {@link #setForStOptions(ForStOptionsFactory)}.
+ */
+@Experimental
+public class ForStStateBackend extends AbstractManagedMemoryStateBackend
+        implements ConfigurableStateBackend {
+
+    private static final long serialVersionUID = 1L;
+
+    private static final Logger LOG = LoggerFactory.getLogger(ForStStateBackend.class);
+
+    /** The number of (re)tries for loading the ForSt JNI library. */
+    private static final int FORST_LIB_LOADING_ATTEMPTS = 3;
+
+    /** Flag whether the native library has been loaded. */
+    private static boolean forStInitialized = false;
+
+    // ------------------------------------------------------------------------
+
+    // -- configuration values, set in the application / configuration
+
+    /**
+     * Base paths for ForSt directory, as configured. Null if not yet set, in which case the
+     * configuration values will be used. The configuration defaults to the TaskManager's temp
+     * directories.
+     */
+    @Nullable private File[] localForStDirectories;
+
+    /** The configurable options. */
+    @Nullable private ReadableConfig configurableOptions;
+
+    /** The options factory to create the ForSt options in the cluster. */
+    @Nullable private ForStOptionsFactory forStOptionsFactory;
+
+    /** The configuration for memory settings (pool sizes, etc.). */
+    private final ForStMemoryConfiguration memoryConfiguration;
+
+    /** The default ForSt property-based metrics options. */
+    private final ForStNativeMetricOptions nativeMetricOptions;
+
+    // -- runtime values, set on TaskManager when initializing / using the backend
+
+    /** Base paths for ForSt directory, as initialized. */
+    private transient File[] initializedDbBasePaths;
+
+    /** JobID for uniquifying backup paths. */
+    private transient JobID jobId;
+
+    /** The index of the next directory to be used from {@link #initializedDbBasePaths}. */
+    private transient int nextDirectory;
+
+    /** Whether we already lazily initialized our local storage directories. */
+    private transient boolean isInitialized;
+
+    /** Factory for Write Buffer Manager and Block Cache. */
+    private final ForStMemoryFactory forStMemoryFactory;
+    // ------------------------------------------------------------------------
+
+    /** Creates a new {@code ForStStateBackend} for storing state. */
+    public ForStStateBackend() {
+        this.nativeMetricOptions = new ForStNativeMetricOptions();
+        this.memoryConfiguration = new ForStMemoryConfiguration();
+        this.forStMemoryFactory = ForStMemoryFactory.DEFAULT;
+    }
+
+    /**
+     * Private constructor that creates a re-configured copy of the state backend.
+     *
+     * @param original The state backend to re-configure.
+     * @param config The configuration.
+     * @param classLoader The class loader.
+     */
+    private ForStStateBackend(
+            ForStStateBackend original, ReadableConfig config, ClassLoader classLoader) {
+        this.memoryConfiguration =
+                ForStMemoryConfiguration.fromOtherAndConfiguration(
+                        original.memoryConfiguration, config);
+        this.memoryConfiguration.validate();
+
+        // configure local directories
+        if (original.localForStDirectories != null) {
+            this.localForStDirectories = original.localForStDirectories;
+        } else {
+            final String forStLocalPaths = config.get(ForStOptions.LOCAL_DIRECTORIES);
+            if (forStLocalPaths != null) {
+                String[] directories = forStLocalPaths.split(",|" + File.pathSeparator);
+
+                try {
+                    setLocalDbStoragePaths(directories);
+                } catch (IllegalArgumentException e) {
+                    throw new IllegalConfigurationException(
+                            "Invalid configuration for ForSt state "
+                                    + "backend's local storage directories: "
+                                    + e.getMessage(),
+                            e);
+                }
+            }
+        }
+
+        // configure metric options
+        this.nativeMetricOptions = ForStNativeMetricOptions.fromConfig(config);
+
+        // configurable options
+        this.configurableOptions = mergeConfigurableOptions(original.configurableOptions, config);
+
+        // configure ForSt options factory
+        try {
+            forStOptionsFactory =
+                    configureOptionsFactory(
+                            original.forStOptionsFactory,
+                            config.get(ForStOptions.OPTIONS_FACTORY),
+                            config,
+                            classLoader);
+        } catch (DynamicCodeLoadingException e) {
+            throw new FlinkRuntimeException(e);
+        }
+
+        // configure latency tracking
+        latencyTrackingConfigBuilder = original.latencyTrackingConfigBuilder.configure(config);
+
+        this.forStMemoryFactory = original.forStMemoryFactory;
+    }
+
+    // ------------------------------------------------------------------------
+    //  Reconfiguration
+    // ------------------------------------------------------------------------
+
+    /**
+     * Creates a copy of this state backend that uses the values defined in the configuration for
+     * fields where that were not yet specified in this state backend.
+     *
+     * @param config The configuration.
+     * @param classLoader The class loader.
+     * @return The re-configured variant of the state backend
+     */
+    @Override
+    public ForStStateBackend configure(ReadableConfig config, ClassLoader classLoader) {
+        return new ForStStateBackend(this, config, classLoader);
+    }
+
+    // ------------------------------------------------------------------------
+    //  State backend methods
+    // ------------------------------------------------------------------------
+
+    private void lazyInitializeForJob(
+            Environment env, @SuppressWarnings("unused") String operatorIdentifier)
+            throws IOException {
+
+        if (isInitialized) {
+            return;
+        }
+
+        this.jobId = env.getJobID();
+
+        // initialize the paths where the local ForSt files should be stored
+        if (localForStDirectories == null) {
+            initializedDbBasePaths = new File[] {env.getTaskManagerInfo().getTmpWorkingDirectory()};
+        } else {
+            List<File> dirs = new ArrayList<>(localForStDirectories.length);
+            StringBuilder errorMessage = new StringBuilder();
+
+            for (File f : localForStDirectories) {
+                File testDir = new File(f, UUID.randomUUID().toString());
+                if (!testDir.mkdirs()) {
+                    String msg =
+                            "Local DB files directory '"
+                                    + f
+                                    + "' does not exist and cannot be created. ";
+                    LOG.error(msg);
+                    errorMessage.append(msg);
+                } else {
+                    dirs.add(f);
+                }
+                //noinspection ResultOfMethodCallIgnored
+                testDir.delete();
+            }
+
+            if (dirs.isEmpty()) {
+                throw new IOException("No local storage directories available. " + errorMessage);
+            } else {
+                initializedDbBasePaths = dirs.toArray(new File[0]);
+            }
+        }
+
+        nextDirectory = new Random().nextInt(initializedDbBasePaths.length);
+
+        isInitialized = true;
+    }
+
+    private File getNextStoragePath() {
+        int ni = nextDirectory + 1;
+        ni = ni >= initializedDbBasePaths.length ? 0 : ni;
+        nextDirectory = ni;
+
+        return initializedDbBasePaths[ni];
+    }
+
+    // ------------------------------------------------------------------------
+    //  State holding data structures
+    // ------------------------------------------------------------------------
+
+    public <K> ForStKeyedStateBackend<K> createForStKeyedStateBackend(
+            KeyedStateBackendParameters<K> parameters) throws IOException {
+        Environment env = parameters.getEnv();
+
+        // first, make sure that the ForSt JNI library is loaded
+        // we do this explicitly here to have better error handling
+        String tempDir = env.getTaskManagerInfo().getTmpWorkingDirectory().getAbsolutePath();
+        ensureForStIsLoaded(tempDir);
+
+        // replace all characters that are not legal for filenames with underscore
+        String fileCompatibleIdentifier =
+                parameters.getOperatorIdentifier().replaceAll("[^a-zA-Z0-9\\-]", "_");
+
+        lazyInitializeForJob(env, fileCompatibleIdentifier);
+
+        File instanceBasePath =
+                new File(
+                        getNextStoragePath(),
+                        "job_"
+                                + jobId
+                                + "_op_"
+                                + fileCompatibleIdentifier
+                                + "_uuid_"
+                                + UUID.randomUUID());
+
+        final OpaqueMemoryResource<ForStSharedResources> sharedResources =
+                ForStOperationUtils.allocateSharedCachesIfConfigured(
+                        memoryConfiguration,
+                        env,
+                        parameters.getManagedMemoryFraction(),
+                        LOG,
+                        forStMemoryFactory);
+        if (sharedResources != null) {
+            LOG.info("Obtained shared ForSt cache of size {} bytes", sharedResources.getSize());
+        }
+        final ForStResourceContainer resourceContainer =
+                createOptionsAndResourceContainer(
+                        sharedResources,
+                        instanceBasePath,
+                        nativeMetricOptions.isStatisticsEnabled());
+
+        ForStKeyedStateBackendBuilder<K> builder =
+                new ForStKeyedStateBackendBuilder<>(
+                                resourceContainer,
+                                stateName -> resourceContainer.getColumnOptions(),
+                                parameters.getKeySerializer(),
+                                parameters.getMetricGroup(),
+                                parameters.getStateHandles())
+                        .setNativeMetricOptions(
+                                resourceContainer.getMemoryWatcherOptions(nativeMetricOptions));
+        return builder.build();
+    }
+
+    @Override
+    public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
+            KeyedStateBackendParameters<K> parameters) {
+        throw new UnsupportedOperationException("Don't support createKeyedStateBackend yet");
+    }
+
+    @Override
+    public OperatorStateBackend createOperatorStateBackend(
+            OperatorStateBackendParameters parameters) throws Exception {
+        // the default for ForSt; eventually there can be a operator state backend based on
+        // ForSt, too.
+        final boolean asyncSnapshots = true;
+        return new DefaultOperatorStateBackendBuilder(
+                        parameters.getEnv().getUserCodeClassLoader().asClassLoader(),
+                        parameters.getEnv().getExecutionConfig(),
+                        asyncSnapshots,
+                        parameters.getStateHandles(),
+                        parameters.getCancelStreamRegistry())
+                .build();
+    }
+
+    private ForStOptionsFactory configureOptionsFactory(
+            @Nullable ForStOptionsFactory originalOptionsFactory,
+            @Nullable String factoryClassName,
+            ReadableConfig config,
+            ClassLoader classLoader)
+            throws DynamicCodeLoadingException {
+
+        ForStOptionsFactory optionsFactory = null;
+
+        if (originalOptionsFactory != null) {
+            if (originalOptionsFactory instanceof ConfigurableForStOptionsFactory) {
+                originalOptionsFactory =
+                        ((ConfigurableForStOptionsFactory) originalOptionsFactory)
+                                .configure(config);
+            }
+            LOG.info("Using application-defined options factory: {}.", originalOptionsFactory);
+
+            optionsFactory = originalOptionsFactory;
+        } else if (factoryClassName != null) {
+            try {
+                Class<? extends ForStOptionsFactory> clazz =
+                        Class.forName(factoryClassName, false, classLoader)
+                                .asSubclass(ForStOptionsFactory.class);
+
+                optionsFactory = clazz.newInstance();
+                if (optionsFactory instanceof ConfigurableForStOptionsFactory) {
+                    optionsFactory =
+                            ((ConfigurableForStOptionsFactory) optionsFactory).configure(config);
+                }
+                LOG.info("Using configured options factory: {}.", optionsFactory);
+
+            } catch (ClassNotFoundException e) {
+                throw new DynamicCodeLoadingException(
+                        "Cannot find configured options factory class: " + factoryClassName, e);
+            } catch (ClassCastException | InstantiationException | IllegalAccessException e) {
+                throw new DynamicCodeLoadingException(
+                        "The class configured under '"
+                                + ForStOptions.OPTIONS_FACTORY.key()
+                                + "' is not a valid options factory ("
+                                + factoryClassName
+                                + ')',
+                        e);
+            }
+        }
+
+        return optionsFactory;
+    }
+
+    // ------------------------------------------------------------------------
+    //  Parameters
+    // ------------------------------------------------------------------------
+
+    /**
+     * Sets the path where the ForSt local files should be stored on the local file system. Setting
+     * this path overrides the default behavior, where the files are stored across the configured
+     * temp directories.
+     *
+     * <p>Passing {@code null} to this function restores the default behavior, where the configured
+     * temp directories will be used.
+     *
+     * @param path The path where the local ForSt database files are stored.
+     */
+    public void setLocalDbStoragePath(String path) {
+        setLocalDbStoragePaths(path == null ? null : new String[] {path});
+    }
+
+    /**
+     * Sets the local directories in which the ForSt database puts some files (like metadata files).
+     * These directories do not need to be persistent, they can be ephemeral, meaning that they are
+     * lost on a machine failure, because state in ForSt is persisted in checkpoints.
+     *
+     * <p>If nothing is configured, these directories default to the TaskManager's local temporary
+     * file directories.
+     *
+     * <p>Each distinct state will be stored in one path, but when the state backend creates
+     * multiple states, they will store their files on different paths.
+     *
+     * <p>Passing {@code null} to this function restores the default behavior, where the configured
+     * temp directories will be used.
+     *
+     * @param paths The paths across which the local ForSt database files will be spread.
+     */
+    public void setLocalDbStoragePaths(String... paths) {
+        if (paths == null) {
+            localForStDirectories = null;
+        } else if (paths.length == 0) {
+            throw new IllegalArgumentException("empty paths");
+        } else {
+            File[] pp = new File[paths.length];
+
+            for (int i = 0; i < paths.length; i++) {
+                final String rawPath = paths[i];
+                final String path;
+
+                if (rawPath == null) {
+                    throw new IllegalArgumentException("null path");
+                } else {
+                    // we need this for backwards compatibility, to allow URIs like 'file:///'...
+                    URI uri = null;
+                    try {
+                        uri = new Path(rawPath).toUri();
+                    } catch (Exception e) {
+                        // cannot parse as a path
+                    }
+
+                    if (uri != null && uri.getScheme() != null) {
+                        if ("file".equalsIgnoreCase(uri.getScheme())) {
+                            path = uri.getPath();
+                        } else {
+                            throw new IllegalArgumentException(
+                                    "Path " + rawPath + " has a non-local scheme");
+                        }
+                    } else {
+                        path = rawPath;
+                    }
+                }
+
+                pp[i] = new File(path);
+                if (!pp[i].isAbsolute()) {
+                    throw new IllegalArgumentException("Relative paths are not supported");
+                }
+            }
+
+            localForStDirectories = pp;
+        }
+    }
+
+    /**
+     * Gets the configured local DB storage paths, or null, if none were configured.
+     *
+     * <p>Under these directories on the TaskManager, ForSt stores some metadata files. These
+     * directories do not need to be persistent, they can be ephermeral, meaning that they are lost
+     * on a machine failure, because state in ForSt is persisted in checkpoints.
+     *
+     * <p>If nothing is configured, these directories default to the TaskManager's local temporary
+     * file directories.
+     */
+    public String[] getLocalDbStoragePaths() {
+        if (localForStDirectories == null) {
+            return null;
+        } else {
+            String[] paths = new String[localForStDirectories.length];
+            for (int i = 0; i < paths.length; i++) {
+                paths[i] = localForStDirectories[i].toString();
+            }
+            return paths;
+        }
+    }
+
+    // ------------------------------------------------------------------------
+    //  Parametrize with ForSt Options
+    // ------------------------------------------------------------------------
+
+    /**
+     * Sets {@link org.rocksdb.Options} for the ForSt instances. Because the options are not
+     * serializable and hold native code references, they must be specified through a factory.
+     *
+     * <p>The options created by the factory here are applied on top of user-configured options from
+     * configuration set by {@link #configure(ReadableConfig, ClassLoader)} with keys in {@link
+     * ForStConfigurableOptions}.
+     *
+     * @param optionsFactory The options factory that lazily creates the ForSt options.
+     */
+    public void setForStOptions(ForStOptionsFactory optionsFactory) {
+        this.forStOptionsFactory = optionsFactory;
+    }
+
+    /** Gets {@link org.rocksdb.Options} for the ForSt instances. */
+    @Nullable
+    public ForStOptionsFactory getForStOptions() {
+        return forStOptionsFactory;
+    }
+
+    // ------------------------------------------------------------------------
+    //  utilities
+    // ------------------------------------------------------------------------
+
+    private ReadableConfig mergeConfigurableOptions(ReadableConfig base, ReadableConfig onTop) {
+        if (base == null) {
+            base = new Configuration();
+        }
+        Configuration configuration = new Configuration();
+        for (ConfigOption<?> option : ForStConfigurableOptions.CANDIDATE_CONFIGS) {
+            Optional<?> baseValue = base.getOptional(option);
+            Optional<?> topValue = onTop.getOptional(option);
+
+            if (topValue.isPresent() || baseValue.isPresent()) {
+                Object validValue = topValue.isPresent() ? topValue.get() : baseValue.get();
+                ForStConfigurableOptions.checkArgumentValid(option, validValue);
+                configuration.setString(option.key(), validValue.toString());
+            }
+        }
+        return configuration;
+    }
+
+    @VisibleForTesting
+    ForStResourceContainer createOptionsAndResourceContainer(@Nullable File instanceBasePath) {
+        return createOptionsAndResourceContainer(null, instanceBasePath, false);
+    }
+
+    @VisibleForTesting
+    private ForStResourceContainer createOptionsAndResourceContainer(
+            @Nullable OpaqueMemoryResource<ForStSharedResources> sharedResources,
+            @Nullable File instanceBasePath,
+            boolean enableStatistics) {
+
+        return new ForStResourceContainer(
+                configurableOptions != null ? configurableOptions : new Configuration(),
+                forStOptionsFactory,
+                sharedResources,
+                instanceBasePath,
+                enableStatistics);
+    }
+
+    @Override
+    public String toString() {
+        return "ForStStateBackend{"
+                + ", localForStDirectories="
+                + Arrays.toString(localForStDirectories)
+                + '}';
+    }
+
+    // ------------------------------------------------------------------------
+    //  static library loading utilities
+    // ------------------------------------------------------------------------
+
+    @VisibleForTesting
+    static void ensureForStIsLoaded(String tempDirectory) throws IOException {
+        ensureForStIsLoaded(tempDirectory, NativeLibraryLoader::getInstance);
+    }
+
+    @VisibleForTesting
+    static void setForStInitialized(boolean initialized) {
+        forStInitialized = initialized;
+    }
+
+    @VisibleForTesting
+    static void ensureForStIsLoaded(
+            String tempDirectory, Supplier<NativeLibraryLoader> nativeLibraryLoaderSupplier)
+            throws IOException {
+        synchronized (ForStStateBackend.class) {
+            if (!forStInitialized) {
+
+                final File tempDirParent = new File(tempDirectory).getAbsoluteFile();
+                LOG.info(
+                        "Attempting to load ForSt native library and store it under '{}'",
+                        tempDirParent);
+
+                Throwable lastException = null;
+                for (int attempt = 1; attempt <= FORST_LIB_LOADING_ATTEMPTS; attempt++) {
+                    File rocksLibFolder = null;
+                    try {
+                        // when multiple instances of this class and ForSt exist in different
+                        // class loaders, then we can see the following exception:
+                        // "java.lang.UnsatisfiedLinkError: Native Library
+                        // /path/to/temp/dir/librocksdbjni-linux64.so
+                        // already loaded in another class loader"
+
+                        // to avoid that, we need to add a random element to the library file path
+                        // (I know, seems like an unnecessary hack, since the JVM obviously can
+                        // handle multiple
+                        //  instances of the same JNI library being loaded in different class
+                        // loaders, but
+                        //  apparently not when coming from the same file path, so there we go)
+
+                        rocksLibFolder = new File(tempDirParent, "rocksdb-lib-" + new AbstractID());
+
+                        // make sure the temp path exists
+                        LOG.debug(
+                                "Attempting to create ForSt native library folder {}",
+                                rocksLibFolder);
+                        // noinspection ResultOfMethodCallIgnored
+                        rocksLibFolder.mkdirs();
+
+                        // explicitly load the JNI dependency if it has not been loaded before
+                        nativeLibraryLoaderSupplier
+                                .get()
+                                .loadLibrary(rocksLibFolder.getAbsolutePath());
+
+                        // this initialization here should validate that the loading succeeded
+                        RocksDB.loadLibrary();
+
+                        // seems to have worked
+                        LOG.info("Successfully loaded ForSt native library");
+                        forStInitialized = true;
+                        return;
+                    } catch (Throwable t) {
+                        lastException = t;
+                        LOG.debug("ForSt JNI library loading attempt {} failed", attempt, t);
+
+                        // try to force ForSt to attempt reloading the library
+                        try {
+                            resetForStLoadedFlag();
+                        } catch (Throwable tt) {
+                            LOG.debug(
+                                    "Failed to reset 'initialized' flag in ForSt native code loader",
+                                    tt);
+                        }
+
+                        FileUtils.deleteDirectoryQuietly(rocksLibFolder);
+                    }
+                }
+
+                throw new IOException("Could not load the native ForSt library", lastException);
+            }
+        }
+    }
+
+    @VisibleForTesting
+    static void resetForStLoadedFlag() throws Exception {
+        final Field initField = NativeLibraryLoader.class.getDeclaredField("initialized");
+        initField.setAccessible(true);
+        initField.setBoolean(null, false);
+    }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackendFactory.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackendFactory.java
new file mode 100644
index 00000000000..4a9b5199f29
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackendFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.runtime.state.StateBackendFactory;
+
+/** A factory that creates an {@link ForStStateBackend} from a configuration. */
+@Experimental
+public class ForStStateBackendFactory implements StateBackendFactory<ForStStateBackend> {
+    @Override
+    public ForStStateBackend createFromConfig(ReadableConfig config, ClassLoader classLoader)
+            throws IllegalConfigurationException {
+        return new ForStStateBackend().configure(config, classLoader);
+    }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStInitITCase.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStInitITCase.java
new file mode 100644
index 00000000000..37868ffa2ac
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStInitITCase.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.runtime.operators.testutils.ExpectedTestException;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.fail;
+
+/** Tests for {@link ForStStateBackend} on initialization. */
+public class ForStInitITCase {
+
+    @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    /**
+     * This test checks that the ForSt native code loader still responds to resetting the init flag.
+     */
+    @Test
+    public void testResetInitFlag() throws Exception {
+        ForStStateBackend.resetForStLoadedFlag();
+    }
+
+    @Test
+    public void testTempLibFolderDeletedOnFail() throws Exception {
+        ForStStateBackend.setForStInitialized(false);
+        File tempFolder = temporaryFolder.newFolder();
+        try {
+            ForStStateBackend.ensureForStIsLoaded(
+                    tempFolder.getAbsolutePath(),
+                    () -> {
+                        throw new ExpectedTestException();
+                    });
+            fail("Not throwing expected exception.");
+        } catch (IOException ignored) {
+            // ignored
+        }
+        File[] files = tempFolder.listFiles();
+        Assert.assertNotNull(files);
+        Assert.assertEquals(0, files.length);
+    }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStMultiClassLoaderTest.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStMultiClassLoaderTest.java
new file mode 100644
index 00000000000..9465cc0e3f3
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStMultiClassLoaderTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.util.FlinkUserCodeClassLoaders;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.rocksdb.RocksDB;
+
+import java.lang.reflect.Method;
+import java.net.URL;
+
+import static org.apache.flink.util.FlinkUserCodeClassLoader.NOOP_EXCEPTION_HANDLER;
+import static org.junit.Assert.assertNotEquals;
+
+/**
+ * This test validates that the ForSt JNI library loading works properly in the presence of the
+ * ForSt code being loaded dynamically via reflection. That can happen when ForSt is in the user
+ * code JAR, or in certain test setups. TODO: test working with both ForSt and RocksDB
+ */
+public class ForStMultiClassLoaderTest {
+
+    @Rule public final TemporaryFolder tmp = new TemporaryFolder();
+
+    @Test
+    public void testTwoSeparateClassLoaders() throws Exception {
+        // collect the libraries / class folders with ForSt related code: the state backend and
+        // ForSt itself
+        final URL codePath1 =
+                ForStStateBackend.class.getProtectionDomain().getCodeSource().getLocation();
+        final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation();
+
+        final ClassLoader parent = getClass().getClassLoader();
+        final ClassLoader loader1 =
+                FlinkUserCodeClassLoaders.childFirst(
+                        new URL[] {codePath1, codePath2},
+                        parent,
+                        new String[0],
+                        NOOP_EXCEPTION_HANDLER,
+                        true);
+        final ClassLoader loader2 =
+                FlinkUserCodeClassLoaders.childFirst(
+                        new URL[] {codePath1, codePath2},
+                        parent,
+                        new String[0],
+                        NOOP_EXCEPTION_HANDLER,
+                        true);
+
+        final String className = ForStStateBackend.class.getName();
+
+        final Class<?> clazz1 = Class.forName(className, false, loader1);
+        final Class<?> clazz2 = Class.forName(className, false, loader2);
+        assertNotEquals(
+                "Test broken - the two reflectively loaded classes are equal", clazz1, clazz2);
+
+        final Object instance1 = clazz1.getConstructor().newInstance();
+        final Object instance2 = clazz2.getConstructor().newInstance();
+
+        final String tempDir = tmp.newFolder().getAbsolutePath();
+
+        final Method meth1 = clazz1.getDeclaredMethod("ensureForStIsLoaded", String.class);
+        final Method meth2 = clazz2.getDeclaredMethod("ensureForStIsLoaded", String.class);
+        meth1.setAccessible(true);
+        meth2.setAccessible(true);
+
+        // if all is well, these methods can both complete successfully
+        meth1.invoke(instance1, tempDir);
+        meth2.invoke(instance2, tempDir);
+    }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendConfigTest.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendConfigTest.java
new file mode 100644
index 00000000000..11733f0ceca
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendConfigTest.java
@@ -0,0 +1,767 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.typeutils.base.IntSerializer;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.operators.testutils.MockEnvironment;
+import org.apache.flink.runtime.query.KvStateRegistry;
+import org.apache.flink.runtime.query.TaskKvStateRegistry;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateBackendParametersImpl;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
+import org.apache.flink.testutils.junit.FailsInGHAContainerWithRootUser;
+import org.apache.flink.util.FileUtils;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.jupiter.api.Timeout;
+import org.junit.rules.TemporaryFolder;
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.BloomFilter;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.CompactionStyle;
+import org.rocksdb.CompressionType;
+import org.rocksdb.DBOptions;
+import org.rocksdb.FlushOptions;
+import org.rocksdb.InfoLogLevel;
+import org.rocksdb.util.SizeUnit;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.apache.flink.state.forst.ForStTestUtils.createKeyedStateBackend;
+import static org.hamcrest.CoreMatchers.anyOf;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/** Tests for configuring the ForSt State Backend. */
+public class ForStStateBackendConfigTest {
+
+    @Rule public final TemporaryFolder tempFolder = new TemporaryFolder();
+
+    // ------------------------------------------------------------------------
+    //  default values
+    // ------------------------------------------------------------------------
+
+    @Test
+    public void testDefaultDbLogDir() throws Exception {
+        final ForStStateBackend backend = new ForStStateBackend();
+        final File logFile = File.createTempFile(getClass().getSimpleName() + "-", ".log");
+        // set the environment variable 'log.file' with the Flink log file location
+        System.setProperty("log.file", logFile.getPath());
+        try (ForStResourceContainer container = backend.createOptionsAndResourceContainer(null)) {
+            assertEquals(
+                    ForStConfigurableOptions.LOG_LEVEL.defaultValue(),
+                    container.getDbOptions().infoLogLevel());
+            assertEquals(logFile.getParent(), container.getDbOptions().dbLogDir());
+        } finally {
+            logFile.delete();
+        }
+
+        StringBuilder longInstanceBasePath =
+                new StringBuilder(tempFolder.newFolder().getAbsolutePath());
+        while (longInstanceBasePath.length() < 255) {
+            longInstanceBasePath.append("/append-for-long-path");
+        }
+        try (ForStResourceContainer container =
+                backend.createOptionsAndResourceContainer(
+                        new File(longInstanceBasePath.toString()))) {
+            assertTrue(container.getDbOptions().dbLogDir().isEmpty());
+        } finally {
+            logFile.delete();
+        }
+    }
+
+    // ------------------------------------------------------------------------
+    //  ForSt local file directory
+    // ------------------------------------------------------------------------
+
+    /** This test checks the behavior for basic setting of local DB directories. */
+    @Test
+    public void testSetDbPath() throws Exception {
+        final ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+        final String testDir1 = tempFolder.newFolder().getAbsolutePath();
+        final String testDir2 = tempFolder.newFolder().getAbsolutePath();
+
+        assertNull(forStStateBackend.getLocalDbStoragePaths());
+
+        forStStateBackend.setLocalDbStoragePath(testDir1);
+        assertArrayEquals(new String[] {testDir1}, forStStateBackend.getLocalDbStoragePaths());
+
+        forStStateBackend.setLocalDbStoragePath(null);
+        assertNull(forStStateBackend.getLocalDbStoragePaths());
+
+        forStStateBackend.setLocalDbStoragePaths(testDir1, testDir2);
+        assertArrayEquals(
+                new String[] {testDir1, testDir2}, forStStateBackend.getLocalDbStoragePaths());
+
+        final MockEnvironment env = getMockEnvironment(tempFolder.newFolder());
+        final ForStKeyedStateBackend<Integer> keyedBackend =
+                createKeyedStateBackend(forStStateBackend, env, IntSerializer.INSTANCE);
+
+        try {
+            File instanceBasePath = keyedBackend.getInstanceBasePath();
+            assertThat(
+                    instanceBasePath.getAbsolutePath(),
+                    anyOf(startsWith(testDir1), startsWith(testDir2)));
+
+            //noinspection NullArgumentToVariableArgMethod
+            forStStateBackend.setLocalDbStoragePaths(null);
+            assertNull(forStStateBackend.getLocalDbStoragePaths());
+        } finally {
+            keyedBackend.dispose();
+            env.close();
+        }
+    }
+
+    @Test
+    public void testStoragePathWithFilePrefix() throws Exception {
+        final File folder = tempFolder.newFolder();
+        final String dbStoragePath = new Path(folder.toURI().toString()).toString();
+
+        assertTrue(dbStoragePath.startsWith("file:"));
+
+        testLocalDbPaths(dbStoragePath, folder);
+    }
+
+    @Test
+    public void testWithDefaultFsSchemeNoStoragePath() throws Exception {
+        try {
+            // set the default file system scheme
+            Configuration config = new Configuration();
+            config.set(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "file:///mydomain.com:8020/flink");
+            FileSystem.initialize(config);
+            testLocalDbPaths(null, tempFolder.getRoot());
+        } finally {
+            FileSystem.initialize(new Configuration());
+        }
+    }
+
+    @Test
+    public void testWithDefaultFsSchemeAbsoluteStoragePath() throws Exception {
+        final File folder = tempFolder.newFolder();
+        final String dbStoragePath = folder.getAbsolutePath();
+
+        try {
+            // set the default file system scheme
+            Configuration config = new Configuration();
+            config.set(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "file:///mydomain.com:8020/flink");
+            FileSystem.initialize(config);
+
+            testLocalDbPaths(dbStoragePath, folder);
+        } finally {
+            FileSystem.initialize(new Configuration());
+        }
+    }
+
+    private void testLocalDbPaths(String configuredPath, File expectedPath) throws Exception {
+        final ForStStateBackend forStBackend = new ForStStateBackend();
+        forStBackend.setLocalDbStoragePath(configuredPath);
+
+        final MockEnvironment env = getMockEnvironment(tempFolder.newFolder());
+        ForStKeyedStateBackend<Integer> keyedBackend =
+                createKeyedStateBackend(forStBackend, env, IntSerializer.INSTANCE);
+
+        try {
+            File instanceBasePath = keyedBackend.getInstanceBasePath();
+            assertThat(
+                    instanceBasePath.getAbsolutePath(), startsWith(expectedPath.getAbsolutePath()));
+
+            //noinspection NullArgumentToVariableArgMethod
+            forStBackend.setLocalDbStoragePaths(null);
+            assertNull(forStBackend.getLocalDbStoragePaths());
+        } finally {
+            keyedBackend.dispose();
+            env.close();
+        }
+    }
+
+    /** Validates that empty arguments for the local DB path are invalid. */
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetEmptyPaths() throws Exception {
+        ForStStateBackend forStStateBackend = new ForStStateBackend();
+        forStStateBackend.setLocalDbStoragePaths();
+    }
+
+    /** Validates that schemes other than 'file:/' are not allowed. */
+    @Test(expected = IllegalArgumentException.class)
+    public void testNonFileSchemePath() throws Exception {
+        ForStStateBackend forStStateBackend = new ForStStateBackend();
+        forStStateBackend.setLocalDbStoragePath("hdfs:///some/path/to/perdition");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testDbPathRelativePaths() throws Exception {
+        ForStStateBackend forStStateBackend = new ForStStateBackend();
+        forStStateBackend.setLocalDbStoragePath("relative/path");
+    }
+
+    @Test
+    @Timeout(value = 60)
+    public void testCleanRelocatedDbLogs() throws Exception {
+        final File folder = tempFolder.newFolder();
+        final File relocatedDBLogDir = tempFolder.newFolder("db_logs");
+        final File logFile = new File(relocatedDBLogDir, "taskManager.log");
+        Files.createFile(logFile.toPath());
+        System.setProperty("log.file", logFile.getAbsolutePath());
+
+        Configuration conf = new Configuration();
+        conf.set(ForStConfigurableOptions.LOG_LEVEL, InfoLogLevel.DEBUG_LEVEL);
+        conf.set(ForStConfigurableOptions.LOG_FILE_NUM, 4);
+        conf.set(ForStConfigurableOptions.LOG_MAX_FILE_SIZE, MemorySize.parse("1kb"));
+        final ForStStateBackend rocksDbBackend =
+                new ForStStateBackend().configure(conf, getClass().getClassLoader());
+        final String dbStoragePath = new Path(folder.toURI().toString()).toString();
+        rocksDbBackend.setLocalDbStoragePath(dbStoragePath);
+
+        final MockEnvironment env = getMockEnvironment(tempFolder.newFolder());
+        ForStKeyedStateBackend<Integer> keyedBackend =
+                createKeyedStateBackend(rocksDbBackend, env, IntSerializer.INSTANCE);
+
+        File instanceBasePath = keyedBackend.getInstanceBasePath();
+        File instanceRocksDBPath = new File(instanceBasePath, "db");
+
+        // avoid tests without relocate.
+        Assume.assumeTrue(instanceRocksDBPath.getAbsolutePath().length() <= 255 - "_LOG".length());
+
+        java.nio.file.Path[] relocatedDbLogs;
+        try {
+            relocatedDbLogs = FileUtils.listDirectory(relocatedDBLogDir.toPath());
+            while (relocatedDbLogs.length <= 2) {
+                // If the default number of log files in rocksdb is not enough, add more logs.
+                try (FlushOptions flushOptions = new FlushOptions()) {
+                    keyedBackend.db.put(RandomUtils.nextBytes(32), RandomUtils.nextBytes(512));
+                    keyedBackend.db.flush(flushOptions);
+                }
+                relocatedDbLogs = FileUtils.listDirectory(relocatedDBLogDir.toPath());
+            }
+        } finally {
+            keyedBackend.dispose();
+            env.close();
+        }
+
+        relocatedDbLogs = FileUtils.listDirectory(relocatedDBLogDir.toPath());
+        assertEquals(1, relocatedDbLogs.length);
+        assertEquals("taskManager.log", relocatedDbLogs[0].toFile().getName());
+    }
+
+    // ------------------------------------------------------------------------
+    //  RocksDB local file automatic from temp directories
+    // ------------------------------------------------------------------------
+
+    /**
+     * This tests whether the RocksDB backends uses the temp directories that are provided from the
+     * {@link Environment} when no db storage path is set.
+     */
+    @Test
+    public void testUseTempDirectories() throws Exception {
+        ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+        File dir1 = tempFolder.newFolder();
+
+        assertNull(forStStateBackend.getLocalDbStoragePaths());
+
+        final MockEnvironment env = getMockEnvironment(dir1);
+        JobID jobID = env.getJobID();
+        KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
+        TaskKvStateRegistry kvStateRegistry = env.getTaskKvStateRegistry();
+        CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
+        ForStKeyedStateBackend<Integer> keyedBackend =
+                forStStateBackend.createForStKeyedStateBackend(
+                        new KeyedStateBackendParametersImpl<>(
+                                env,
+                                jobID,
+                                "test_op",
+                                IntSerializer.INSTANCE,
+                                1,
+                                keyGroupRange,
+                                kvStateRegistry,
+                                TtlTimeProvider.DEFAULT,
+                                new UnregisteredMetricsGroup(),
+                                Collections.emptyList(),
+                                cancelStreamRegistry));
+
+        try {
+            File instanceBasePath = keyedBackend.getInstanceBasePath();
+            assertThat(instanceBasePath.getAbsolutePath(), startsWith(dir1.getAbsolutePath()));
+        } finally {
+            keyedBackend.dispose();
+            env.close();
+        }
+    }
+
+    // ------------------------------------------------------------------------
+    //  RocksDB local file directory initialization
+    // ------------------------------------------------------------------------
+
+    @Test
+    @Category(FailsInGHAContainerWithRootUser.class)
+    public void testFailWhenNoLocalStorageDir() throws Exception {
+        final File targetDir = tempFolder.newFolder();
+        Assume.assumeTrue(
+                "Cannot mark directory non-writable", targetDir.setWritable(false, false));
+
+        ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+        try (MockEnvironment env = getMockEnvironment(tempFolder.newFolder())) {
+            forStStateBackend.setLocalDbStoragePath(targetDir.getAbsolutePath());
+
+            boolean hasFailure = false;
+            try {
+                JobID jobID = env.getJobID();
+                KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
+                TaskKvStateRegistry kvStateRegistry =
+                        new KvStateRegistry().createTaskRegistry(env.getJobID(), new JobVertexID());
+                CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
+                forStStateBackend.createForStKeyedStateBackend(
+                        new KeyedStateBackendParametersImpl<>(
+                                env,
+                                jobID,
+                                "foobar",
+                                IntSerializer.INSTANCE,
+                                1,
+                                keyGroupRange,
+                                kvStateRegistry,
+                                TtlTimeProvider.DEFAULT,
+                                new UnregisteredMetricsGroup(),
+                                Collections.emptyList(),
+                                cancelStreamRegistry));
+            } catch (Exception e) {
+                assertTrue(e.getMessage().contains("No local storage directories available"));
+                assertTrue(e.getMessage().contains(targetDir.getAbsolutePath()));
+                hasFailure = true;
+            }
+            assertTrue(
+                    "We must see a failure because no storaged directory is feasible.", hasFailure);
+        } finally {
+            //noinspection ResultOfMethodCallIgnored
+            targetDir.setWritable(true, false);
+        }
+    }
+
+    @Test
+    public void testContinueOnSomeDbDirectoriesMissing() throws Exception {
+        final File targetDir1 = tempFolder.newFolder();
+        final File targetDir2 = tempFolder.newFolder();
+        Assume.assumeTrue(
+                "Cannot mark directory non-writable", targetDir1.setWritable(false, false));
+
+        ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+        try (MockEnvironment env = getMockEnvironment(tempFolder.newFolder())) {
+            forStStateBackend.setLocalDbStoragePaths(
+                    targetDir1.getAbsolutePath(), targetDir2.getAbsolutePath());
+
+            try {
+                JobID jobID = env.getJobID();
+                KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
+                TaskKvStateRegistry kvStateRegistry =
+                        new KvStateRegistry().createTaskRegistry(env.getJobID(), new JobVertexID());
+                CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
+                ForStKeyedStateBackend<Integer> keyedStateBackend =
+                        forStStateBackend.createForStKeyedStateBackend(
+                                new KeyedStateBackendParametersImpl<>(
+                                        env,
+                                        jobID,
+                                        "foobar",
+                                        IntSerializer.INSTANCE,
+                                        1,
+                                        keyGroupRange,
+                                        kvStateRegistry,
+                                        TtlTimeProvider.DEFAULT,
+                                        new UnregisteredMetricsGroup(),
+                                        Collections.emptyList(),
+                                        cancelStreamRegistry));
+
+                keyedStateBackend.dispose();
+            } catch (Exception e) {
+                e.printStackTrace();
+                fail("Backend initialization failed even though some paths were available");
+            }
+        } finally {
+            //noinspection ResultOfMethodCallIgnored
+            targetDir1.setWritable(true, false);
+        }
+    }
+
+    // ------------------------------------------------------------------------
+    //  RocksDB Options
+    // ------------------------------------------------------------------------
+    @Test
+    public void testConfigurableOptionsFromConfig() throws Exception {
+        Configuration configuration = new Configuration();
+
+        // verify illegal configuration
+        {
+            verifyIllegalArgument(ForStConfigurableOptions.MAX_BACKGROUND_THREADS, "-1");
+            verifyIllegalArgument(ForStConfigurableOptions.LOG_LEVEL, "DEBUG");
+            verifyIllegalArgument(ForStConfigurableOptions.LOG_DIR, "tmp/rocksdb-logs/");
+            verifyIllegalArgument(ForStConfigurableOptions.LOG_DIR, "");
+            verifyIllegalArgument(ForStConfigurableOptions.LOG_FILE_NUM, "0");
+            verifyIllegalArgument(ForStConfigurableOptions.LOG_FILE_NUM, "-1");
+            verifyIllegalArgument(ForStConfigurableOptions.LOG_MAX_FILE_SIZE, "-1KB");
+            verifyIllegalArgument(ForStConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "-1");
+            verifyIllegalArgument(ForStConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "-1");
+
+            verifyIllegalArgument(ForStConfigurableOptions.TARGET_FILE_SIZE_BASE, "0KB");
+            verifyIllegalArgument(ForStConfigurableOptions.MAX_SIZE_LEVEL_BASE, "1BB");
+            verifyIllegalArgument(ForStConfigurableOptions.WRITE_BUFFER_SIZE, "-1KB");
+            verifyIllegalArgument(ForStConfigurableOptions.BLOCK_SIZE, "0MB");
+            verifyIllegalArgument(ForStConfigurableOptions.METADATA_BLOCK_SIZE, "0MB");
+            verifyIllegalArgument(ForStConfigurableOptions.BLOCK_CACHE_SIZE, "0");
+
+            verifyIllegalArgument(ForStConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "1");
+
+            verifyIllegalArgument(ForStConfigurableOptions.COMPACTION_STYLE, "LEV");
+            verifyIllegalArgument(ForStConfigurableOptions.COMPRESSION_PER_LEVEL, "SNAP");
+            verifyIllegalArgument(ForStConfigurableOptions.USE_BLOOM_FILTER, "NO");
+            verifyIllegalArgument(ForStConfigurableOptions.BLOOM_FILTER_BLOCK_BASED_MODE, "YES");
+        }
+
+        // verify legal configuration
+        {
+            configuration.setString(ForStConfigurableOptions.LOG_LEVEL.key(), "DEBUG_LEVEL");
+            configuration.setString(ForStConfigurableOptions.LOG_DIR.key(), "/tmp/rocksdb-logs/");
+            configuration.setString(ForStConfigurableOptions.LOG_FILE_NUM.key(), "10");
+            configuration.setString(ForStConfigurableOptions.LOG_MAX_FILE_SIZE.key(), "2MB");
+            configuration.setString(ForStConfigurableOptions.COMPACTION_STYLE.key(), "level");
+            configuration.setString(
+                    ForStConfigurableOptions.COMPRESSION_PER_LEVEL.key(),
+                    "no_compression;snappy_compression;lz4_compression");
+            configuration.setString(ForStConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE.key(), "TRUE");
+            configuration.setString(ForStConfigurableOptions.TARGET_FILE_SIZE_BASE.key(), "8 mb");
+            configuration.setString(ForStConfigurableOptions.MAX_SIZE_LEVEL_BASE.key(), "128MB");
+            configuration.setString(ForStConfigurableOptions.MAX_BACKGROUND_THREADS.key(), "4");
+            configuration.setString(ForStConfigurableOptions.MAX_WRITE_BUFFER_NUMBER.key(), "4");
+            configuration.setString(
+                    ForStConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE.key(), "2");
+            configuration.setString(ForStConfigurableOptions.WRITE_BUFFER_SIZE.key(), "64 MB");
+            configuration.setString(ForStConfigurableOptions.BLOCK_SIZE.key(), "4 kb");
+            configuration.setString(ForStConfigurableOptions.METADATA_BLOCK_SIZE.key(), "8 kb");
+            configuration.setString(ForStConfigurableOptions.BLOCK_CACHE_SIZE.key(), "512 mb");
+            configuration.setString(ForStConfigurableOptions.USE_BLOOM_FILTER.key(), "TRUE");
+
+            try (ForStResourceContainer optionsContainer =
+                    new ForStResourceContainer(configuration, null, null, null, false)) {
+
+                DBOptions dbOptions = optionsContainer.getDbOptions();
+                assertEquals(-1, dbOptions.maxOpenFiles());
+                assertEquals(InfoLogLevel.DEBUG_LEVEL, dbOptions.infoLogLevel());
+                assertEquals("/tmp/rocksdb-logs/", dbOptions.dbLogDir());
+                assertEquals(10, dbOptions.keepLogFileNum());
+                assertEquals(2 * SizeUnit.MB, dbOptions.maxLogFileSize());
+
+                ColumnFamilyOptions columnOptions = optionsContainer.getColumnOptions();
+                assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
+                assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
+                assertEquals(8 * SizeUnit.MB, columnOptions.targetFileSizeBase());
+                assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
+                assertEquals(4, columnOptions.maxWriteBufferNumber());
+                assertEquals(2, columnOptions.minWriteBufferNumberToMerge());
+                assertEquals(64 * SizeUnit.MB, columnOptions.writeBufferSize());
+                assertEquals(
+                        Arrays.asList(
+                                CompressionType.NO_COMPRESSION,
+                                CompressionType.SNAPPY_COMPRESSION,
+                                CompressionType.LZ4_COMPRESSION),
+                        columnOptions.compressionPerLevel());
+
+                BlockBasedTableConfig tableConfig =
+                        (BlockBasedTableConfig) columnOptions.tableFormatConfig();
+                assertEquals(4 * SizeUnit.KB, tableConfig.blockSize());
+                assertEquals(8 * SizeUnit.KB, tableConfig.metadataBlockSize());
+                assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
+                assertTrue(tableConfig.filterPolicy() instanceof BloomFilter);
+            }
+        }
+    }
+
+    @Test
+    public void testOptionsFactory() throws Exception {
+        ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+        // verify that user-defined options factory could be configured via config.yaml
+        Configuration config = new Configuration();
+        config.setString(ForStOptions.OPTIONS_FACTORY.key(), TestOptionsFactory.class.getName());
+        config.setString(TestOptionsFactory.BACKGROUND_JOBS_OPTION.key(), "4");
+
+        forStStateBackend = forStStateBackend.configure(config, getClass().getClassLoader());
+
+        assertTrue(forStStateBackend.getForStOptions() instanceof TestOptionsFactory);
+
+        try (ForStResourceContainer optionsContainer =
+                forStStateBackend.createOptionsAndResourceContainer(null)) {
+            DBOptions dbOptions = optionsContainer.getDbOptions();
+            assertEquals(4, dbOptions.maxBackgroundJobs());
+        }
+
+        // verify that user-defined options factory could be set programmatically and override
+        // pre-configured one.
+        forStStateBackend.setForStOptions(
+                new ForStOptionsFactory() {
+                    @Override
+                    public DBOptions createDBOptions(
+                            DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
+                        return currentOptions;
+                    }
+
+                    @Override
+                    public ColumnFamilyOptions createColumnOptions(
+                            ColumnFamilyOptions currentOptions,
+                            Collection<AutoCloseable> handlesToClose) {
+                        return currentOptions.setCompactionStyle(CompactionStyle.FIFO);
+                    }
+                });
+
+        try (ForStResourceContainer optionsContainer =
+                forStStateBackend.createOptionsAndResourceContainer(null)) {
+            ColumnFamilyOptions colCreated = optionsContainer.getColumnOptions();
+            assertEquals(CompactionStyle.FIFO, colCreated.compactionStyle());
+        }
+    }
+
+    @Test
+    public void testConfigurableOptions() throws Exception {
+        Configuration configuration = new Configuration();
+        configuration.set(ForStConfigurableOptions.COMPACTION_STYLE, CompactionStyle.UNIVERSAL);
+        try (final ForStResourceContainer optionsContainer =
+                new ForStResourceContainer(configuration, null, null, null, false)) {
+
+            final ColumnFamilyOptions columnFamilyOptions = optionsContainer.getColumnOptions();
+            assertNotNull(columnFamilyOptions);
+            assertEquals(CompactionStyle.UNIVERSAL, columnFamilyOptions.compactionStyle());
+        }
+
+        try (final ForStResourceContainer optionsContainer =
+                new ForStResourceContainer(new Configuration(), null, null, null, false)) {
+
+            final ColumnFamilyOptions columnFamilyOptions = optionsContainer.getColumnOptions();
+            assertNotNull(columnFamilyOptions);
+            assertEquals(CompactionStyle.LEVEL, columnFamilyOptions.compactionStyle());
+        }
+    }
+
+    @Test
+    public void testPredefinedAndOptionsFactory() throws Exception {
+        final ForStOptionsFactory optionsFactory =
+                new ForStOptionsFactory() {
+                    @Override
+                    public DBOptions createDBOptions(
+                            DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
+                        return currentOptions;
+                    }
+
+                    @Override
+                    public ColumnFamilyOptions createColumnOptions(
+                            ColumnFamilyOptions currentOptions,
+                            Collection<AutoCloseable> handlesToClose) {
+                        return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
+                    }
+                };
+
+        try (final ForStResourceContainer optionsContainer =
+                new ForStResourceContainer(optionsFactory)) {
+
+            final ColumnFamilyOptions columnFamilyOptions = optionsContainer.getColumnOptions();
+            assertNotNull(columnFamilyOptions);
+            assertEquals(CompactionStyle.UNIVERSAL, columnFamilyOptions.compactionStyle());
+        }
+    }
+
+    // ------------------------------------------------------------------------
+    //  Reconfiguration
+    // ------------------------------------------------------------------------
+
+    @Test
+    public void testForStReconfigurationCopiesExistingValues() throws Exception {
+        final ForStStateBackend original = new ForStStateBackend();
+        final ForStOptionsFactory optionsFactory = new TestOptionsFactory();
+        original.setForStOptions(optionsFactory);
+
+        final String[] localDirs =
+                new String[] {
+                    tempFolder.newFolder().getAbsolutePath(),
+                    tempFolder.newFolder().getAbsolutePath()
+                };
+        original.setLocalDbStoragePaths(localDirs);
+
+        ForStStateBackend copy =
+                original.configure(
+                        new Configuration(), Thread.currentThread().getContextClassLoader());
+
+        assertArrayEquals(original.getLocalDbStoragePaths(), copy.getLocalDbStoragePaths());
+        assertEquals(original.getForStOptions(), copy.getForStOptions());
+    }
+
+    // ------------------------------------------------------------------------
+    //  RocksDB Memory Control
+    // ------------------------------------------------------------------------
+
+    @Test
+    public void testDefaultMemoryControlParameters() {
+        ForStMemoryConfiguration memSettings = new ForStMemoryConfiguration();
+        assertTrue(memSettings.isUsingManagedMemory());
+        assertFalse(memSettings.isUsingFixedMemoryPerSlot());
+        assertEquals(
+                ForStOptions.HIGH_PRIORITY_POOL_RATIO.defaultValue(),
+                memSettings.getHighPriorityPoolRatio(),
+                0.0);
+        assertEquals(
+                ForStOptions.WRITE_BUFFER_RATIO.defaultValue(),
+                memSettings.getWriteBufferRatio(),
+                0.0);
+
+        ForStMemoryConfiguration configured =
+                ForStMemoryConfiguration.fromOtherAndConfiguration(
+                        memSettings, new Configuration());
+        assertTrue(configured.isUsingManagedMemory());
+        assertFalse(configured.isUsingFixedMemoryPerSlot());
+        assertEquals(
+                ForStOptions.HIGH_PRIORITY_POOL_RATIO.defaultValue(),
+                configured.getHighPriorityPoolRatio(),
+                0.0);
+        assertEquals(
+                ForStOptions.WRITE_BUFFER_RATIO.defaultValue(),
+                configured.getWriteBufferRatio(),
+                0.0);
+    }
+
+    @Test
+    public void testConfigureManagedMemory() {
+        final Configuration config = new Configuration();
+        config.set(ForStOptions.USE_MANAGED_MEMORY, true);
+
+        final ForStMemoryConfiguration memSettings =
+                ForStMemoryConfiguration.fromOtherAndConfiguration(
+                        new ForStMemoryConfiguration(), config);
+
+        assertTrue(memSettings.isUsingManagedMemory());
+    }
+
+    @Test
+    public void testConfigureIllegalMemoryControlParameters() {
+        ForStMemoryConfiguration memSettings = new ForStMemoryConfiguration();
+
+        verifySetParameter(() -> memSettings.setFixedMemoryPerSlot("-1B"));
+        verifySetParameter(() -> memSettings.setHighPriorityPoolRatio(-0.1));
+        verifySetParameter(() -> memSettings.setHighPriorityPoolRatio(1.1));
+        verifySetParameter(() -> memSettings.setWriteBufferRatio(-0.1));
+        verifySetParameter(() -> memSettings.setWriteBufferRatio(1.1));
+
+        memSettings.setFixedMemoryPerSlot("128MB");
+        memSettings.setWriteBufferRatio(0.6);
+        memSettings.setHighPriorityPoolRatio(0.6);
+
+        try {
+            // sum of writeBufferRatio and highPriPoolRatio larger than 1.0
+            memSettings.validate();
+            fail("Expected an IllegalArgumentException.");
+        } catch (IllegalArgumentException expected) {
+            // expected exception
+        }
+    }
+
+    private void verifySetParameter(Runnable setter) {
+        try {
+            setter.run();
+            fail("No expected IllegalArgumentException.");
+        } catch (IllegalArgumentException expected) {
+            // expected exception
+        }
+    }
+
+    // ------------------------------------------------------------------------
+    //  Utilities
+    // ------------------------------------------------------------------------
+
+    static MockEnvironment getMockEnvironment(File tempDir) {
+        return MockEnvironment.builder()
+                .setUserCodeClassLoader(ForStStateBackendConfigTest.class.getClassLoader())
+                .setTaskManagerRuntimeInfo(
+                        new TestingTaskManagerRuntimeInfo(new Configuration(), tempDir))
+                .build();
+    }
+
+    private void verifyIllegalArgument(ConfigOption<?> configOption, String configValue) {
+        Configuration configuration = new Configuration();
+        configuration.setString(configOption.key(), configValue);
+
+        ForStStateBackend stateBackend = new ForStStateBackend();
+        try {
+            stateBackend.configure(configuration, null);
+            fail("Not throwing expected IllegalArgumentException.");
+        } catch (IllegalArgumentException e) {
+            // ignored
+        }
+    }
+
+    /** An implementation of options factory for testing. */
+    public static class TestOptionsFactory implements ConfigurableForStOptionsFactory {
+        public static final ConfigOption<Integer> BACKGROUND_JOBS_OPTION =
+                ConfigOptions.key("my.custom.rocksdb.backgroundJobs").intType().defaultValue(2);
+
+        private int backgroundJobs = BACKGROUND_JOBS_OPTION.defaultValue();
+
+        @Override
+        public DBOptions createDBOptions(
+                DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
+            return currentOptions.setMaxBackgroundJobs(backgroundJobs);
+        }
+
+        @Override
+        public ColumnFamilyOptions createColumnOptions(
+                ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
+            return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
+        }
+
+        @Override
+        public ForStOptionsFactory configure(ReadableConfig configuration) {
+            this.backgroundJobs = configuration.get(BACKGROUND_JOBS_OPTION);
+            return this;
+        }
+    }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendFactoryTest.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendFactoryTest.java
new file mode 100644
index 00000000000..effcd2be793
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendFactoryTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.configuration.CheckpointingOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.StateBackendOptions;
+import org.apache.flink.runtime.state.StateBackend;
+import org.apache.flink.runtime.state.StateBackendLoader;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashSet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/** Tests for the ForStStateBackendFactory. */
+public class ForStStateBackendFactoryTest {
+
+    @Rule public final TemporaryFolder tmp = new TemporaryFolder();
+
+    private final ClassLoader cl = getClass().getClassLoader();
+
+    private final String backendKey = StateBackendOptions.STATE_BACKEND.key();
+
+    // ------------------------------------------------------------------------
+
+    @Test
+    public void testEmbeddedFactoryName() {
+        // construct the name such that it will not be automatically adjusted on refactorings
+        String factoryName = "org.apache.flink.state.forst.For";
+        factoryName += "StStateBackendFactory";
+
+        // !!! if this fails, the code in StateBackendLoader must be adjusted
+        assertEquals(factoryName, ForStStateBackendFactory.class.getName());
+    }
+
+    /**
+     * Validates loading a file system state backend with additional parameters from the cluster
+     * configuration.
+     */
+    @Test
+    public void testLoadForStStateBackend() throws Exception {
+        final String localDir1 = tmp.newFolder().getAbsolutePath();
+        final String localDir2 = tmp.newFolder().getAbsolutePath();
+        final String localDirs = localDir1 + File.pathSeparator + localDir2;
+        final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();
+
+        // TODO: Support short name of backendKey
+
+        final Configuration config1 = new Configuration();
+        config1.setString(backendKey, ForStStateBackendFactory.class.getName());
+        config1.set(ForStOptions.LOCAL_DIRECTORIES, localDirs);
+        config1.set(CheckpointingOptions.INCREMENTAL_CHECKPOINTS, incremental);
+
+        StateBackend backend1 = StateBackendLoader.loadStateBackendFromConfig(config1, cl, null);
+
+        assertTrue(backend1 instanceof ForStStateBackend);
+
+        ForStStateBackend fs1 = (ForStStateBackend) backend1;
+
+        checkPaths(fs1.getLocalDbStoragePaths(), localDir1, localDir2);
+    }
+
+    /**
+     * Validates taking the application-defined ForSt state backend and adding with additional
+     * parameters from the cluster configuration, but giving precedence to application-defined
+     * parameters over configuration-defined parameters.
+     */
+    @Test
+    public void testLoadForStStateBackendMixed() throws Exception {
+        final String localDir1 = tmp.newFolder().getAbsolutePath();
+        final String localDir2 = tmp.newFolder().getAbsolutePath();
+        final String localDir3 = tmp.newFolder().getAbsolutePath();
+        final String localDir4 = tmp.newFolder().getAbsolutePath();
+
+        final ForStStateBackend backend = new ForStStateBackend();
+        backend.setLocalDbStoragePaths(localDir1, localDir2);
+
+        final Configuration config = new Configuration();
+        config.setString(backendKey, "hashmap"); // this should not be picked up
+        config.set(
+                ForStOptions.LOCAL_DIRECTORIES,
+                localDir3 + ":" + localDir4); // this should not be picked up
+
+        final StateBackend loadedBackend =
+                StateBackendLoader.fromApplicationOrConfigOrDefault(
+                        backend, new Configuration(), config, cl, null);
+        assertTrue(loadedBackend instanceof ForStStateBackend);
+
+        final ForStStateBackend loadedRocks = (ForStStateBackend) loadedBackend;
+
+        checkPaths(loadedRocks.getLocalDbStoragePaths(), localDir1, localDir2);
+    }
+
+    // ------------------------------------------------------------------------
+
+    private static void checkPaths(String[] pathsArray, String... paths) {
+        assertNotNull(pathsArray);
+        assertNotNull(paths);
+
+        assertEquals(pathsArray.length, paths.length);
+
+        HashSet<String> pathsSet = new HashSet<>(Arrays.asList(pathsArray));
+
+        for (String path : paths) {
+            assertTrue(pathsSet.contains(path));
+        }
+    }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStTestUtils.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStTestUtils.java
new file mode 100644
index 00000000000..e5cb4c91872
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStTestUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateBackendParametersImpl;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+
+import java.io.IOException;
+import java.util.Collections;
+
+/** Test utils for the ForSt state backend. */
+public final class ForStTestUtils {
+    public static <K> ForStKeyedStateBackend<K> createKeyedStateBackend(
+            ForStStateBackend forStStateBackend, Environment env, TypeSerializer<K> keySerializer)
+            throws IOException {
+
+        return forStStateBackend.createForStKeyedStateBackend(
+                new KeyedStateBackendParametersImpl<>(
+                        env,
+                        env.getJobID(),
+                        "test_op",
+                        keySerializer,
+                        1,
+                        new KeyGroupRange(0, 0),
+                        env.getTaskKvStateRegistry(),
+                        TtlTimeProvider.DEFAULT,
+                        new UnregisteredMetricsGroup(),
+                        (name, value) -> {},
+                        Collections.emptyList(),
+                        new CloseableRegistry(),
+                        1.0));
+    }
+}