You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ha...@apache.org on 2024/04/26 01:48:56 UTC
(flink) 06/07: [FLINK-35047][state] Introduce ForStStateBackend and ForStKeyedStateBackend
This is an automated email from the ASF dual-hosted git repository.
hangxiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 417484e6029c879a38b698434b9a2f8ba94413fb
Author: Hangxiang Yu <ma...@gmail.com>
AuthorDate: Tue Apr 16 23:34:36 2024 +0800
[FLINK-35047][state] Introduce ForStStateBackend and ForStKeyedStateBackend
---
.../flink/state/forst/ForStKeyedStateBackend.java | 134 ++++
.../state/forst/ForStKeyedStateBackendBuilder.java | 187 +++++
.../flink/state/forst/ForStStateBackend.java | 655 ++++++++++++++++++
.../state/forst/ForStStateBackendFactory.java | 34 +
.../apache/flink/state/forst/ForStInitITCase.java | 64 ++
.../state/forst/ForStMultiClassLoaderTest.java | 88 +++
.../state/forst/ForStStateBackendConfigTest.java | 767 +++++++++++++++++++++
.../state/forst/ForStStateBackendFactoryTest.java | 132 ++++
.../apache/flink/state/forst/ForStTestUtils.java | 54 ++
9 files changed, 2115 insertions(+)
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackend.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackend.java
new file mode 100644
index 00000000000..8c7dfd0ba0f
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackend.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.Preconditions;
+
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.RocksDB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * A KeyedStateBackend that stores its state in {@code ForSt}. This state backend can store very
+ * large state that exceeds memory even disk to remote storage. TODO: Support to implement the new
+ * interface of KeyedStateBackend
+ */
+public class ForStKeyedStateBackend<K> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ForStKeyedStateBackend.class);
+
+ /** The key serializer. */
+ protected final TypeSerializer<K> keySerializer;
+
+ /** The container of ForSt options. */
+ private final ForStResourceContainer optionsContainer;
+
+ /** Path where this configured instance stores its local data directory. */
+ private final File localBasePath;
+
+ /**
+ * We are not using the default column family for Flink state ops, but we still need to remember
+ * this handle so that we can close it properly when the backend is closed. Note that the one
+ * returned by {@link RocksDB#open(String)} is different from that by {@link
+ * RocksDB#getDefaultColumnFamily()}, probably it's a bug of RocksDB java API.
+ */
+ private final ColumnFamilyHandle defaultColumnFamily;
+
+ /** The native metrics monitor. */
+ private final ForStNativeMetricMonitor nativeMetricMonitor;
+
+ /**
+ * Our ForSt database. The different k/v states that we have don't each have their own ForSt
+ * instance. They all write to this instance but to their own column family.
+ */
+ protected final RocksDB db;
+
+ // mark whether this backend is already disposed and prevent duplicate disposing
+ private boolean disposed = false;
+
+ public ForStKeyedStateBackend(
+ File localBasePath,
+ ForStResourceContainer optionsContainer,
+ TypeSerializer<K> keySerializer,
+ RocksDB db,
+ ColumnFamilyHandle defaultColumnFamilyHandle,
+ ForStNativeMetricMonitor nativeMetricMonitor) {
+
+ this.localBasePath = Preconditions.checkNotNull(localBasePath);
+ this.optionsContainer = Preconditions.checkNotNull(optionsContainer);
+ this.keySerializer = keySerializer;
+ this.db = db;
+ this.defaultColumnFamily = defaultColumnFamilyHandle;
+ this.nativeMetricMonitor = nativeMetricMonitor;
+ }
+
+ /** Should only be called by one thread, and only after all accesses to the DB happened. */
+ public void dispose() {
+ if (this.disposed) {
+ return;
+ }
+
+ // IMPORTANT: null reference to signal potential async checkpoint workers that the db was
+ // disposed, as
+ // working on the disposed object results in SEGFAULTS.
+ if (db != null) {
+
+ // Metric collection occurs on a background thread. When this method returns
+ // it is guaranteed that thr ForSt reference has been invalidated
+ // and no more metric collection will be attempted against the database.
+ if (nativeMetricMonitor != null) {
+ nativeMetricMonitor.close();
+ }
+
+ IOUtils.closeQuietly(defaultColumnFamily);
+
+ // ... and finally close the DB instance ...
+ IOUtils.closeQuietly(db);
+
+ IOUtils.closeQuietly(optionsContainer);
+
+ cleanLocalBasePath();
+ }
+ this.disposed = true;
+ }
+
+ private void cleanLocalBasePath() {
+ LOG.info(
+ "Closed ForSt State Backend. Cleaning up ForSt local working directory {}.",
+ localBasePath);
+
+ try {
+ FileUtils.deleteDirectory(localBasePath);
+ } catch (IOException ex) {
+ LOG.warn("Could not delete ForSt local working directory: {}", localBasePath, ex);
+ }
+ }
+
+ @VisibleForTesting
+ File getInstanceBasePath() {
+ return localBasePath;
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackendBuilder.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackendBuilder.java
new file mode 100644
index 00000000000..1ff68f413a4
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStKeyedStateBackendBuilder.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.state.BackendBuildingException;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.StateBackendBuilder;
+import org.apache.flink.runtime.state.StateSerializerProvider;
+import org.apache.flink.state.forst.restore.ForStNoneRestoreOperation;
+import org.apache.flink.state.forst.restore.ForStRestoreOperation;
+import org.apache.flink.state.forst.restore.ForStRestoreResult;
+import org.apache.flink.util.CollectionUtil;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.Preconditions;
+
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.function.Function;
+
+/**
+ * Builder class for {@link ForStKeyedStateBackend} which handles all necessary initializations and
+ * cleanups.
+ *
+ * @param <K> The data type that the key serializer serializes.
+ */
+public class ForStKeyedStateBackendBuilder<K>
+ implements StateBackendBuilder<ForStKeyedStateBackend<K>, BackendBuildingException> {
+
+ protected final Logger logger = LoggerFactory.getLogger(getClass());
+
+ private final StateSerializerProvider<K> keySerializerProvider;
+ private final Collection<KeyedStateHandle> restoreStateHandles;
+
+ /** Factory function to create column family options from state name. */
+ private final Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory;
+
+ /** The container of ForSt option factory and predefined options. */
+ private final ForStResourceContainer optionsContainer;
+
+ /** Path where this configured instance stores its data directory. */
+ private final File localBasePath;
+
+ /** Path where this configured instance stores its ForSt database. */
+ private final File localForStPath;
+
+ private final MetricGroup metricGroup;
+
+ /** ForSt property-based and statistics-based native metrics options. */
+ private ForStNativeMetricOptions nativeMetricOptions;
+
+ public ForStKeyedStateBackendBuilder(
+ ForStResourceContainer optionsContainer,
+ Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
+ TypeSerializer<K> keySerializer,
+ MetricGroup metricGroup,
+ @Nonnull Collection<KeyedStateHandle> stateHandles) {
+ this.optionsContainer = optionsContainer;
+ this.localBasePath = optionsContainer.getLocalBasePath();
+ this.localForStPath = optionsContainer.getLocalForStPath();
+ this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);
+ this.keySerializerProvider =
+ StateSerializerProvider.fromNewRegisteredSerializer(keySerializer);
+ this.metricGroup = metricGroup;
+ this.restoreStateHandles = stateHandles;
+ this.nativeMetricOptions = new ForStNativeMetricOptions();
+ }
+
+ ForStKeyedStateBackendBuilder<K> setNativeMetricOptions(
+ ForStNativeMetricOptions nativeMetricOptions) {
+ this.nativeMetricOptions = nativeMetricOptions;
+ return this;
+ }
+
+ @Override
+ public ForStKeyedStateBackend<K> build() throws BackendBuildingException {
+ ColumnFamilyHandle defaultColumnFamilyHandle = null;
+ ForStNativeMetricMonitor nativeMetricMonitor = null;
+ RocksDB db = null;
+ ForStRestoreOperation restoreOperation = null;
+
+ try {
+ prepareLocalDirectories();
+ restoreOperation = getForStRestoreOperation();
+ ForStRestoreResult restoreResult = restoreOperation.restore();
+ db = restoreResult.getDb();
+ defaultColumnFamilyHandle = restoreResult.getDefaultColumnFamilyHandle();
+ nativeMetricMonitor = restoreResult.getNativeMetricMonitor();
+
+ // TODO: Support to init snapshot strategy
+
+ } catch (Throwable e) {
+ // Do clean up
+ IOUtils.closeQuietly(defaultColumnFamilyHandle);
+ IOUtils.closeQuietly(nativeMetricMonitor);
+ IOUtils.closeQuietly(db);
+ // it's possible that db has been initialized but later restore steps failed
+ IOUtils.closeQuietly(restoreOperation);
+ IOUtils.closeQuietly(optionsContainer);
+ try {
+ FileUtils.deleteDirectory(localBasePath);
+ } catch (Exception ex) {
+ logger.warn("Failed to delete local base path for ForSt: " + localBasePath, ex);
+ }
+ // Log and rethrow
+ if (e instanceof BackendBuildingException) {
+ throw (BackendBuildingException) e;
+ } else {
+ String errMsg = "Caught unexpected exception.";
+ logger.error(errMsg, e);
+ throw new BackendBuildingException(errMsg, e);
+ }
+ }
+ logger.info(
+ "Finished building ForSt keyed state-backend at local base path: {}.",
+ localBasePath);
+ return new ForStKeyedStateBackend<>(
+ this.localBasePath,
+ this.optionsContainer,
+ this.keySerializerProvider.currentSchemaSerializer(),
+ db,
+ defaultColumnFamilyHandle,
+ nativeMetricMonitor);
+ }
+
+ private ForStRestoreOperation getForStRestoreOperation() {
+ DBOptions dbOptions = optionsContainer.getDbOptions();
+ if (CollectionUtil.isEmptyOrAllElementsNull(restoreStateHandles)) {
+ return new ForStNoneRestoreOperation(
+ localForStPath,
+ dbOptions,
+ columnFamilyOptionsFactory,
+ nativeMetricOptions,
+ metricGroup);
+ }
+ // TODO: Support Restoring
+ throw new UnsupportedOperationException("Not support restoring yet for ForStStateBackend");
+ }
+
+ private void prepareLocalDirectories() throws IOException {
+ checkAndCreateDirectory(localBasePath);
+ if (localForStPath.exists()) {
+ // Clear the base directory when the backend is created
+ // in case something crashed and the backend never reached dispose()
+ FileUtils.deleteDirectory(localBasePath);
+ }
+ }
+
+ private static void checkAndCreateDirectory(File directory) throws IOException {
+ if (directory.exists()) {
+ if (!directory.isDirectory()) {
+ throw new IOException("Not a directory: " + directory);
+ }
+ } else if (!directory.mkdirs()) {
+ throw new IOException(
+ String.format("Could not create ForSt data directory at %s.", directory));
+ }
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackend.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackend.java
new file mode 100644
index 00000000000..be21b54db18
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackend.java
@@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.memory.OpaqueMemoryResource;
+import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
+import org.apache.flink.runtime.state.AbstractManagedMemoryStateBackend;
+import org.apache.flink.runtime.state.ConfigurableStateBackend;
+import org.apache.flink.runtime.state.DefaultOperatorStateBackendBuilder;
+import org.apache.flink.runtime.state.OperatorStateBackend;
+import org.apache.flink.state.forst.ForStMemoryControllerUtils.ForStMemoryFactory;
+import org.apache.flink.util.AbstractID;
+import org.apache.flink.util.DynamicCodeLoadingException;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkRuntimeException;
+
+import org.rocksdb.NativeLibraryLoader;
+import org.rocksdb.RocksDB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.Random;
+import java.util.UUID;
+import java.util.function.Supplier;
+
+/**
+ * A {@link org.apache.flink.runtime.state.StateBackend} that stores its state in a ForSt instance.
+ * This state backend can store very large state that exceeds memory even disk and spills to remote
+ * storage.
+ *
+ * <p>The behavior of the ForSt instances can be parametrized by setting ForSt Options using the
+ * methods {@link #setForStOptions(ForStOptionsFactory)}.
+ */
+@Experimental
+public class ForStStateBackend extends AbstractManagedMemoryStateBackend
+ implements ConfigurableStateBackend {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(ForStStateBackend.class);
+
+ /** The number of (re)tries for loading the ForSt JNI library. */
+ private static final int FORST_LIB_LOADING_ATTEMPTS = 3;
+
+ /** Flag whether the native library has been loaded. */
+ private static boolean forStInitialized = false;
+
+ // ------------------------------------------------------------------------
+
+ // -- configuration values, set in the application / configuration
+
+ /**
+ * Base paths for ForSt directory, as configured. Null if not yet set, in which case the
+ * configuration values will be used. The configuration defaults to the TaskManager's temp
+ * directories.
+ */
+ @Nullable private File[] localForStDirectories;
+
+ /** The configurable options. */
+ @Nullable private ReadableConfig configurableOptions;
+
+ /** The options factory to create the ForSt options in the cluster. */
+ @Nullable private ForStOptionsFactory forStOptionsFactory;
+
+ /** The configuration for memory settings (pool sizes, etc.). */
+ private final ForStMemoryConfiguration memoryConfiguration;
+
+ /** The default ForSt property-based metrics options. */
+ private final ForStNativeMetricOptions nativeMetricOptions;
+
+ // -- runtime values, set on TaskManager when initializing / using the backend
+
+ /** Base paths for ForSt directory, as initialized. */
+ private transient File[] initializedDbBasePaths;
+
+ /** JobID for uniquifying backup paths. */
+ private transient JobID jobId;
+
+ /** The index of the next directory to be used from {@link #initializedDbBasePaths}. */
+ private transient int nextDirectory;
+
+ /** Whether we already lazily initialized our local storage directories. */
+ private transient boolean isInitialized;
+
+ /** Factory for Write Buffer Manager and Block Cache. */
+ private final ForStMemoryFactory forStMemoryFactory;
+ // ------------------------------------------------------------------------
+
+ /** Creates a new {@code ForStStateBackend} for storing state. */
+ public ForStStateBackend() {
+ this.nativeMetricOptions = new ForStNativeMetricOptions();
+ this.memoryConfiguration = new ForStMemoryConfiguration();
+ this.forStMemoryFactory = ForStMemoryFactory.DEFAULT;
+ }
+
+ /**
+ * Private constructor that creates a re-configured copy of the state backend.
+ *
+ * @param original The state backend to re-configure.
+ * @param config The configuration.
+ * @param classLoader The class loader.
+ */
+ private ForStStateBackend(
+ ForStStateBackend original, ReadableConfig config, ClassLoader classLoader) {
+ this.memoryConfiguration =
+ ForStMemoryConfiguration.fromOtherAndConfiguration(
+ original.memoryConfiguration, config);
+ this.memoryConfiguration.validate();
+
+ // configure local directories
+ if (original.localForStDirectories != null) {
+ this.localForStDirectories = original.localForStDirectories;
+ } else {
+ final String forStLocalPaths = config.get(ForStOptions.LOCAL_DIRECTORIES);
+ if (forStLocalPaths != null) {
+ String[] directories = forStLocalPaths.split(",|" + File.pathSeparator);
+
+ try {
+ setLocalDbStoragePaths(directories);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalConfigurationException(
+ "Invalid configuration for ForSt state "
+ + "backend's local storage directories: "
+ + e.getMessage(),
+ e);
+ }
+ }
+ }
+
+ // configure metric options
+ this.nativeMetricOptions = ForStNativeMetricOptions.fromConfig(config);
+
+ // configurable options
+ this.configurableOptions = mergeConfigurableOptions(original.configurableOptions, config);
+
+ // configure ForSt options factory
+ try {
+ forStOptionsFactory =
+ configureOptionsFactory(
+ original.forStOptionsFactory,
+ config.get(ForStOptions.OPTIONS_FACTORY),
+ config,
+ classLoader);
+ } catch (DynamicCodeLoadingException e) {
+ throw new FlinkRuntimeException(e);
+ }
+
+ // configure latency tracking
+ latencyTrackingConfigBuilder = original.latencyTrackingConfigBuilder.configure(config);
+
+ this.forStMemoryFactory = original.forStMemoryFactory;
+ }
+
+ // ------------------------------------------------------------------------
+ // Reconfiguration
+ // ------------------------------------------------------------------------
+
+ /**
+ * Creates a copy of this state backend that uses the values defined in the configuration for
+ * fields where that were not yet specified in this state backend.
+ *
+ * @param config The configuration.
+ * @param classLoader The class loader.
+ * @return The re-configured variant of the state backend
+ */
+ @Override
+ public ForStStateBackend configure(ReadableConfig config, ClassLoader classLoader) {
+ return new ForStStateBackend(this, config, classLoader);
+ }
+
+ // ------------------------------------------------------------------------
+ // State backend methods
+ // ------------------------------------------------------------------------
+
+ private void lazyInitializeForJob(
+ Environment env, @SuppressWarnings("unused") String operatorIdentifier)
+ throws IOException {
+
+ if (isInitialized) {
+ return;
+ }
+
+ this.jobId = env.getJobID();
+
+ // initialize the paths where the local ForSt files should be stored
+ if (localForStDirectories == null) {
+ initializedDbBasePaths = new File[] {env.getTaskManagerInfo().getTmpWorkingDirectory()};
+ } else {
+ List<File> dirs = new ArrayList<>(localForStDirectories.length);
+ StringBuilder errorMessage = new StringBuilder();
+
+ for (File f : localForStDirectories) {
+ File testDir = new File(f, UUID.randomUUID().toString());
+ if (!testDir.mkdirs()) {
+ String msg =
+ "Local DB files directory '"
+ + f
+ + "' does not exist and cannot be created. ";
+ LOG.error(msg);
+ errorMessage.append(msg);
+ } else {
+ dirs.add(f);
+ }
+ //noinspection ResultOfMethodCallIgnored
+ testDir.delete();
+ }
+
+ if (dirs.isEmpty()) {
+ throw new IOException("No local storage directories available. " + errorMessage);
+ } else {
+ initializedDbBasePaths = dirs.toArray(new File[0]);
+ }
+ }
+
+ nextDirectory = new Random().nextInt(initializedDbBasePaths.length);
+
+ isInitialized = true;
+ }
+
+ private File getNextStoragePath() {
+ int ni = nextDirectory + 1;
+ ni = ni >= initializedDbBasePaths.length ? 0 : ni;
+ nextDirectory = ni;
+
+ return initializedDbBasePaths[ni];
+ }
+
+ // ------------------------------------------------------------------------
+ // State holding data structures
+ // ------------------------------------------------------------------------
+
+ public <K> ForStKeyedStateBackend<K> createForStKeyedStateBackend(
+ KeyedStateBackendParameters<K> parameters) throws IOException {
+ Environment env = parameters.getEnv();
+
+ // first, make sure that the ForSt JNI library is loaded
+ // we do this explicitly here to have better error handling
+ String tempDir = env.getTaskManagerInfo().getTmpWorkingDirectory().getAbsolutePath();
+ ensureForStIsLoaded(tempDir);
+
+ // replace all characters that are not legal for filenames with underscore
+ String fileCompatibleIdentifier =
+ parameters.getOperatorIdentifier().replaceAll("[^a-zA-Z0-9\\-]", "_");
+
+ lazyInitializeForJob(env, fileCompatibleIdentifier);
+
+ File instanceBasePath =
+ new File(
+ getNextStoragePath(),
+ "job_"
+ + jobId
+ + "_op_"
+ + fileCompatibleIdentifier
+ + "_uuid_"
+ + UUID.randomUUID());
+
+ final OpaqueMemoryResource<ForStSharedResources> sharedResources =
+ ForStOperationUtils.allocateSharedCachesIfConfigured(
+ memoryConfiguration,
+ env,
+ parameters.getManagedMemoryFraction(),
+ LOG,
+ forStMemoryFactory);
+ if (sharedResources != null) {
+ LOG.info("Obtained shared ForSt cache of size {} bytes", sharedResources.getSize());
+ }
+ final ForStResourceContainer resourceContainer =
+ createOptionsAndResourceContainer(
+ sharedResources,
+ instanceBasePath,
+ nativeMetricOptions.isStatisticsEnabled());
+
+ ForStKeyedStateBackendBuilder<K> builder =
+ new ForStKeyedStateBackendBuilder<>(
+ resourceContainer,
+ stateName -> resourceContainer.getColumnOptions(),
+ parameters.getKeySerializer(),
+ parameters.getMetricGroup(),
+ parameters.getStateHandles())
+ .setNativeMetricOptions(
+ resourceContainer.getMemoryWatcherOptions(nativeMetricOptions));
+ return builder.build();
+ }
+
+ @Override
+ public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
+ KeyedStateBackendParameters<K> parameters) {
+ throw new UnsupportedOperationException("Don't support createKeyedStateBackend yet");
+ }
+
+ @Override
+ public OperatorStateBackend createOperatorStateBackend(
+ OperatorStateBackendParameters parameters) throws Exception {
+ // the default for ForSt; eventually there can be a operator state backend based on
+ // ForSt, too.
+ final boolean asyncSnapshots = true;
+ return new DefaultOperatorStateBackendBuilder(
+ parameters.getEnv().getUserCodeClassLoader().asClassLoader(),
+ parameters.getEnv().getExecutionConfig(),
+ asyncSnapshots,
+ parameters.getStateHandles(),
+ parameters.getCancelStreamRegistry())
+ .build();
+ }
+
+ private ForStOptionsFactory configureOptionsFactory(
+ @Nullable ForStOptionsFactory originalOptionsFactory,
+ @Nullable String factoryClassName,
+ ReadableConfig config,
+ ClassLoader classLoader)
+ throws DynamicCodeLoadingException {
+
+ ForStOptionsFactory optionsFactory = null;
+
+ if (originalOptionsFactory != null) {
+ if (originalOptionsFactory instanceof ConfigurableForStOptionsFactory) {
+ originalOptionsFactory =
+ ((ConfigurableForStOptionsFactory) originalOptionsFactory)
+ .configure(config);
+ }
+ LOG.info("Using application-defined options factory: {}.", originalOptionsFactory);
+
+ optionsFactory = originalOptionsFactory;
+ } else if (factoryClassName != null) {
+ try {
+ Class<? extends ForStOptionsFactory> clazz =
+ Class.forName(factoryClassName, false, classLoader)
+ .asSubclass(ForStOptionsFactory.class);
+
+ optionsFactory = clazz.newInstance();
+ if (optionsFactory instanceof ConfigurableForStOptionsFactory) {
+ optionsFactory =
+ ((ConfigurableForStOptionsFactory) optionsFactory).configure(config);
+ }
+ LOG.info("Using configured options factory: {}.", optionsFactory);
+
+ } catch (ClassNotFoundException e) {
+ throw new DynamicCodeLoadingException(
+ "Cannot find configured options factory class: " + factoryClassName, e);
+ } catch (ClassCastException | InstantiationException | IllegalAccessException e) {
+ throw new DynamicCodeLoadingException(
+ "The class configured under '"
+ + ForStOptions.OPTIONS_FACTORY.key()
+ + "' is not a valid options factory ("
+ + factoryClassName
+ + ')',
+ e);
+ }
+ }
+
+ return optionsFactory;
+ }
+
+ // ------------------------------------------------------------------------
+ // Parameters
+ // ------------------------------------------------------------------------
+
+ /**
+ * Sets the path where the ForSt local files should be stored on the local file system. Setting
+ * this path overrides the default behavior, where the files are stored across the configured
+ * temp directories.
+ *
+ * <p>Passing {@code null} to this function restores the default behavior, where the configured
+ * temp directories will be used.
+ *
+ * @param path The path where the local ForSt database files are stored.
+ */
+ public void setLocalDbStoragePath(String path) {
+ setLocalDbStoragePaths(path == null ? null : new String[] {path});
+ }
+
+ /**
+ * Sets the local directories in which the ForSt database puts some files (like metadata files).
+ * These directories do not need to be persistent, they can be ephemeral, meaning that they are
+ * lost on a machine failure, because state in ForSt is persisted in checkpoints.
+ *
+ * <p>If nothing is configured, these directories default to the TaskManager's local temporary
+ * file directories.
+ *
+ * <p>Each distinct state will be stored in one path, but when the state backend creates
+ * multiple states, they will store their files on different paths.
+ *
+ * <p>Passing {@code null} to this function restores the default behavior, where the configured
+ * temp directories will be used.
+ *
+ * @param paths The paths across which the local ForSt database files will be spread.
+ */
+ public void setLocalDbStoragePaths(String... paths) {
+ if (paths == null) {
+ localForStDirectories = null;
+ } else if (paths.length == 0) {
+ throw new IllegalArgumentException("empty paths");
+ } else {
+ File[] pp = new File[paths.length];
+
+ for (int i = 0; i < paths.length; i++) {
+ final String rawPath = paths[i];
+ final String path;
+
+ if (rawPath == null) {
+ throw new IllegalArgumentException("null path");
+ } else {
+ // we need this for backwards compatibility, to allow URIs like 'file:///'...
+ URI uri = null;
+ try {
+ uri = new Path(rawPath).toUri();
+ } catch (Exception e) {
+ // cannot parse as a path
+ }
+
+ if (uri != null && uri.getScheme() != null) {
+ if ("file".equalsIgnoreCase(uri.getScheme())) {
+ path = uri.getPath();
+ } else {
+ throw new IllegalArgumentException(
+ "Path " + rawPath + " has a non-local scheme");
+ }
+ } else {
+ path = rawPath;
+ }
+ }
+
+ pp[i] = new File(path);
+ if (!pp[i].isAbsolute()) {
+ throw new IllegalArgumentException("Relative paths are not supported");
+ }
+ }
+
+ localForStDirectories = pp;
+ }
+ }
+
+ /**
+ * Gets the configured local DB storage paths, or null, if none were configured.
+ *
+ * <p>Under these directories on the TaskManager, ForSt stores some metadata files. These
+ * directories do not need to be persistent, they can be ephermeral, meaning that they are lost
+ * on a machine failure, because state in ForSt is persisted in checkpoints.
+ *
+ * <p>If nothing is configured, these directories default to the TaskManager's local temporary
+ * file directories.
+ */
+ public String[] getLocalDbStoragePaths() {
+ if (localForStDirectories == null) {
+ return null;
+ } else {
+ String[] paths = new String[localForStDirectories.length];
+ for (int i = 0; i < paths.length; i++) {
+ paths[i] = localForStDirectories[i].toString();
+ }
+ return paths;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Parametrize with ForSt Options
+ // ------------------------------------------------------------------------
+
+ /**
+ * Sets {@link org.rocksdb.Options} for the ForSt instances. Because the options are not
+ * serializable and hold native code references, they must be specified through a factory.
+ *
+ * <p>The options created by the factory here are applied on top of user-configured options from
+ * configuration set by {@link #configure(ReadableConfig, ClassLoader)} with keys in {@link
+ * ForStConfigurableOptions}.
+ *
+ * @param optionsFactory The options factory that lazily creates the ForSt options.
+ */
+ public void setForStOptions(ForStOptionsFactory optionsFactory) {
+ this.forStOptionsFactory = optionsFactory;
+ }
+
+ /** Gets {@link org.rocksdb.Options} for the ForSt instances. */
+ @Nullable
+ public ForStOptionsFactory getForStOptions() {
+ return forStOptionsFactory;
+ }
+
+ // ------------------------------------------------------------------------
+ // utilities
+ // ------------------------------------------------------------------------
+
+ private ReadableConfig mergeConfigurableOptions(ReadableConfig base, ReadableConfig onTop) {
+ if (base == null) {
+ base = new Configuration();
+ }
+ Configuration configuration = new Configuration();
+ for (ConfigOption<?> option : ForStConfigurableOptions.CANDIDATE_CONFIGS) {
+ Optional<?> baseValue = base.getOptional(option);
+ Optional<?> topValue = onTop.getOptional(option);
+
+ if (topValue.isPresent() || baseValue.isPresent()) {
+ Object validValue = topValue.isPresent() ? topValue.get() : baseValue.get();
+ ForStConfigurableOptions.checkArgumentValid(option, validValue);
+ configuration.setString(option.key(), validValue.toString());
+ }
+ }
+ return configuration;
+ }
+
+ @VisibleForTesting
+ ForStResourceContainer createOptionsAndResourceContainer(@Nullable File instanceBasePath) {
+ return createOptionsAndResourceContainer(null, instanceBasePath, false);
+ }
+
+ @VisibleForTesting
+ private ForStResourceContainer createOptionsAndResourceContainer(
+ @Nullable OpaqueMemoryResource<ForStSharedResources> sharedResources,
+ @Nullable File instanceBasePath,
+ boolean enableStatistics) {
+
+ return new ForStResourceContainer(
+ configurableOptions != null ? configurableOptions : new Configuration(),
+ forStOptionsFactory,
+ sharedResources,
+ instanceBasePath,
+ enableStatistics);
+ }
+
+ @Override
+ public String toString() {
+ return "ForStStateBackend{"
+ + ", localForStDirectories="
+ + Arrays.toString(localForStDirectories)
+ + '}';
+ }
+
+ // ------------------------------------------------------------------------
+ // static library loading utilities
+ // ------------------------------------------------------------------------
+
+ @VisibleForTesting
+ static void ensureForStIsLoaded(String tempDirectory) throws IOException {
+ ensureForStIsLoaded(tempDirectory, NativeLibraryLoader::getInstance);
+ }
+
+ @VisibleForTesting
+ static void setForStInitialized(boolean initialized) {
+ forStInitialized = initialized;
+ }
+
+ @VisibleForTesting
+ static void ensureForStIsLoaded(
+ String tempDirectory, Supplier<NativeLibraryLoader> nativeLibraryLoaderSupplier)
+ throws IOException {
+ synchronized (ForStStateBackend.class) {
+ if (!forStInitialized) {
+
+ final File tempDirParent = new File(tempDirectory).getAbsoluteFile();
+ LOG.info(
+ "Attempting to load ForSt native library and store it under '{}'",
+ tempDirParent);
+
+ Throwable lastException = null;
+ for (int attempt = 1; attempt <= FORST_LIB_LOADING_ATTEMPTS; attempt++) {
+ File rocksLibFolder = null;
+ try {
+ // when multiple instances of this class and ForSt exist in different
+ // class loaders, then we can see the following exception:
+ // "java.lang.UnsatisfiedLinkError: Native Library
+ // /path/to/temp/dir/librocksdbjni-linux64.so
+ // already loaded in another class loader"
+
+ // to avoid that, we need to add a random element to the library file path
+ // (I know, seems like an unnecessary hack, since the JVM obviously can
+ // handle multiple
+ // instances of the same JNI library being loaded in different class
+ // loaders, but
+ // apparently not when coming from the same file path, so there we go)
+
+ rocksLibFolder = new File(tempDirParent, "rocksdb-lib-" + new AbstractID());
+
+ // make sure the temp path exists
+ LOG.debug(
+ "Attempting to create ForSt native library folder {}",
+ rocksLibFolder);
+ // noinspection ResultOfMethodCallIgnored
+ rocksLibFolder.mkdirs();
+
+ // explicitly load the JNI dependency if it has not been loaded before
+ nativeLibraryLoaderSupplier
+ .get()
+ .loadLibrary(rocksLibFolder.getAbsolutePath());
+
+ // this initialization here should validate that the loading succeeded
+ RocksDB.loadLibrary();
+
+ // seems to have worked
+ LOG.info("Successfully loaded ForSt native library");
+ forStInitialized = true;
+ return;
+ } catch (Throwable t) {
+ lastException = t;
+ LOG.debug("ForSt JNI library loading attempt {} failed", attempt, t);
+
+ // try to force ForSt to attempt reloading the library
+ try {
+ resetForStLoadedFlag();
+ } catch (Throwable tt) {
+ LOG.debug(
+ "Failed to reset 'initialized' flag in ForSt native code loader",
+ tt);
+ }
+
+ FileUtils.deleteDirectoryQuietly(rocksLibFolder);
+ }
+ }
+
+ throw new IOException("Could not load the native ForSt library", lastException);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ static void resetForStLoadedFlag() throws Exception {
+ final Field initField = NativeLibraryLoader.class.getDeclaredField("initialized");
+ initField.setAccessible(true);
+ initField.setBoolean(null, false);
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackendFactory.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackendFactory.java
new file mode 100644
index 00000000000..4a9b5199f29
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateBackendFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.runtime.state.StateBackendFactory;
+
+/** A factory that creates an {@link ForStStateBackend} from a configuration. */
+@Experimental
+public class ForStStateBackendFactory implements StateBackendFactory<ForStStateBackend> {
+ @Override
+ public ForStStateBackend createFromConfig(ReadableConfig config, ClassLoader classLoader)
+ throws IllegalConfigurationException {
+ return new ForStStateBackend().configure(config, classLoader);
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStInitITCase.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStInitITCase.java
new file mode 100644
index 00000000000..37868ffa2ac
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStInitITCase.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.runtime.operators.testutils.ExpectedTestException;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.fail;
+
+/** Tests for {@link ForStStateBackend} on initialization. */
+public class ForStInitITCase {
+
+ @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ /**
+ * This test checks that the ForSt native code loader still responds to resetting the init flag.
+ */
+ @Test
+ public void testResetInitFlag() throws Exception {
+ ForStStateBackend.resetForStLoadedFlag();
+ }
+
+ @Test
+ public void testTempLibFolderDeletedOnFail() throws Exception {
+ ForStStateBackend.setForStInitialized(false);
+ File tempFolder = temporaryFolder.newFolder();
+ try {
+ ForStStateBackend.ensureForStIsLoaded(
+ tempFolder.getAbsolutePath(),
+ () -> {
+ throw new ExpectedTestException();
+ });
+ fail("Not throwing expected exception.");
+ } catch (IOException ignored) {
+ // ignored
+ }
+ File[] files = tempFolder.listFiles();
+ Assert.assertNotNull(files);
+ Assert.assertEquals(0, files.length);
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStMultiClassLoaderTest.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStMultiClassLoaderTest.java
new file mode 100644
index 00000000000..9465cc0e3f3
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStMultiClassLoaderTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.util.FlinkUserCodeClassLoaders;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.rocksdb.RocksDB;
+
+import java.lang.reflect.Method;
+import java.net.URL;
+
+import static org.apache.flink.util.FlinkUserCodeClassLoader.NOOP_EXCEPTION_HANDLER;
+import static org.junit.Assert.assertNotEquals;
+
+/**
+ * This test validates that the ForSt JNI library loading works properly in the presence of the
+ * ForSt code being loaded dynamically via reflection. That can happen when ForSt is in the user
+ * code JAR, or in certain test setups. TODO: test working with both ForSt and RocksDB
+ */
+public class ForStMultiClassLoaderTest {
+
+ @Rule public final TemporaryFolder tmp = new TemporaryFolder();
+
+ @Test
+ public void testTwoSeparateClassLoaders() throws Exception {
+ // collect the libraries / class folders with ForSt related code: the state backend and
+ // ForSt itself
+ final URL codePath1 =
+ ForStStateBackend.class.getProtectionDomain().getCodeSource().getLocation();
+ final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation();
+
+ final ClassLoader parent = getClass().getClassLoader();
+ final ClassLoader loader1 =
+ FlinkUserCodeClassLoaders.childFirst(
+ new URL[] {codePath1, codePath2},
+ parent,
+ new String[0],
+ NOOP_EXCEPTION_HANDLER,
+ true);
+ final ClassLoader loader2 =
+ FlinkUserCodeClassLoaders.childFirst(
+ new URL[] {codePath1, codePath2},
+ parent,
+ new String[0],
+ NOOP_EXCEPTION_HANDLER,
+ true);
+
+ final String className = ForStStateBackend.class.getName();
+
+ final Class<?> clazz1 = Class.forName(className, false, loader1);
+ final Class<?> clazz2 = Class.forName(className, false, loader2);
+ assertNotEquals(
+ "Test broken - the two reflectively loaded classes are equal", clazz1, clazz2);
+
+ final Object instance1 = clazz1.getConstructor().newInstance();
+ final Object instance2 = clazz2.getConstructor().newInstance();
+
+ final String tempDir = tmp.newFolder().getAbsolutePath();
+
+ final Method meth1 = clazz1.getDeclaredMethod("ensureForStIsLoaded", String.class);
+ final Method meth2 = clazz2.getDeclaredMethod("ensureForStIsLoaded", String.class);
+ meth1.setAccessible(true);
+ meth2.setAccessible(true);
+
+ // if all is well, these methods can both complete successfully
+ meth1.invoke(instance1, tempDir);
+ meth2.invoke(instance2, tempDir);
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendConfigTest.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendConfigTest.java
new file mode 100644
index 00000000000..11733f0ceca
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendConfigTest.java
@@ -0,0 +1,767 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.typeutils.base.IntSerializer;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.operators.testutils.MockEnvironment;
+import org.apache.flink.runtime.query.KvStateRegistry;
+import org.apache.flink.runtime.query.TaskKvStateRegistry;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateBackendParametersImpl;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
+import org.apache.flink.testutils.junit.FailsInGHAContainerWithRootUser;
+import org.apache.flink.util.FileUtils;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.jupiter.api.Timeout;
+import org.junit.rules.TemporaryFolder;
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.BloomFilter;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.CompactionStyle;
+import org.rocksdb.CompressionType;
+import org.rocksdb.DBOptions;
+import org.rocksdb.FlushOptions;
+import org.rocksdb.InfoLogLevel;
+import org.rocksdb.util.SizeUnit;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.apache.flink.state.forst.ForStTestUtils.createKeyedStateBackend;
+import static org.hamcrest.CoreMatchers.anyOf;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/** Tests for configuring the ForSt State Backend. */
+public class ForStStateBackendConfigTest {
+
+ @Rule public final TemporaryFolder tempFolder = new TemporaryFolder();
+
+ // ------------------------------------------------------------------------
+ // default values
+ // ------------------------------------------------------------------------
+
+ @Test
+ public void testDefaultDbLogDir() throws Exception {
+ final ForStStateBackend backend = new ForStStateBackend();
+ final File logFile = File.createTempFile(getClass().getSimpleName() + "-", ".log");
+ // set the environment variable 'log.file' with the Flink log file location
+ System.setProperty("log.file", logFile.getPath());
+ try (ForStResourceContainer container = backend.createOptionsAndResourceContainer(null)) {
+ assertEquals(
+ ForStConfigurableOptions.LOG_LEVEL.defaultValue(),
+ container.getDbOptions().infoLogLevel());
+ assertEquals(logFile.getParent(), container.getDbOptions().dbLogDir());
+ } finally {
+ logFile.delete();
+ }
+
+ StringBuilder longInstanceBasePath =
+ new StringBuilder(tempFolder.newFolder().getAbsolutePath());
+ while (longInstanceBasePath.length() < 255) {
+ longInstanceBasePath.append("/append-for-long-path");
+ }
+ try (ForStResourceContainer container =
+ backend.createOptionsAndResourceContainer(
+ new File(longInstanceBasePath.toString()))) {
+ assertTrue(container.getDbOptions().dbLogDir().isEmpty());
+ } finally {
+ logFile.delete();
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // ForSt local file directory
+ // ------------------------------------------------------------------------
+
+ /** This test checks the behavior for basic setting of local DB directories. */
+ @Test
+ public void testSetDbPath() throws Exception {
+ final ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+ final String testDir1 = tempFolder.newFolder().getAbsolutePath();
+ final String testDir2 = tempFolder.newFolder().getAbsolutePath();
+
+ assertNull(forStStateBackend.getLocalDbStoragePaths());
+
+ forStStateBackend.setLocalDbStoragePath(testDir1);
+ assertArrayEquals(new String[] {testDir1}, forStStateBackend.getLocalDbStoragePaths());
+
+ forStStateBackend.setLocalDbStoragePath(null);
+ assertNull(forStStateBackend.getLocalDbStoragePaths());
+
+ forStStateBackend.setLocalDbStoragePaths(testDir1, testDir2);
+ assertArrayEquals(
+ new String[] {testDir1, testDir2}, forStStateBackend.getLocalDbStoragePaths());
+
+ final MockEnvironment env = getMockEnvironment(tempFolder.newFolder());
+ final ForStKeyedStateBackend<Integer> keyedBackend =
+ createKeyedStateBackend(forStStateBackend, env, IntSerializer.INSTANCE);
+
+ try {
+ File instanceBasePath = keyedBackend.getInstanceBasePath();
+ assertThat(
+ instanceBasePath.getAbsolutePath(),
+ anyOf(startsWith(testDir1), startsWith(testDir2)));
+
+ //noinspection NullArgumentToVariableArgMethod
+ forStStateBackend.setLocalDbStoragePaths(null);
+ assertNull(forStStateBackend.getLocalDbStoragePaths());
+ } finally {
+ keyedBackend.dispose();
+ env.close();
+ }
+ }
+
+ @Test
+ public void testStoragePathWithFilePrefix() throws Exception {
+ final File folder = tempFolder.newFolder();
+ final String dbStoragePath = new Path(folder.toURI().toString()).toString();
+
+ assertTrue(dbStoragePath.startsWith("file:"));
+
+ testLocalDbPaths(dbStoragePath, folder);
+ }
+
+ @Test
+ public void testWithDefaultFsSchemeNoStoragePath() throws Exception {
+ try {
+ // set the default file system scheme
+ Configuration config = new Configuration();
+ config.set(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "file:///mydomain.com:8020/flink");
+ FileSystem.initialize(config);
+ testLocalDbPaths(null, tempFolder.getRoot());
+ } finally {
+ FileSystem.initialize(new Configuration());
+ }
+ }
+
+ @Test
+ public void testWithDefaultFsSchemeAbsoluteStoragePath() throws Exception {
+ final File folder = tempFolder.newFolder();
+ final String dbStoragePath = folder.getAbsolutePath();
+
+ try {
+ // set the default file system scheme
+ Configuration config = new Configuration();
+ config.set(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "file:///mydomain.com:8020/flink");
+ FileSystem.initialize(config);
+
+ testLocalDbPaths(dbStoragePath, folder);
+ } finally {
+ FileSystem.initialize(new Configuration());
+ }
+ }
+
+ private void testLocalDbPaths(String configuredPath, File expectedPath) throws Exception {
+ final ForStStateBackend forStBackend = new ForStStateBackend();
+ forStBackend.setLocalDbStoragePath(configuredPath);
+
+ final MockEnvironment env = getMockEnvironment(tempFolder.newFolder());
+ ForStKeyedStateBackend<Integer> keyedBackend =
+ createKeyedStateBackend(forStBackend, env, IntSerializer.INSTANCE);
+
+ try {
+ File instanceBasePath = keyedBackend.getInstanceBasePath();
+ assertThat(
+ instanceBasePath.getAbsolutePath(), startsWith(expectedPath.getAbsolutePath()));
+
+ //noinspection NullArgumentToVariableArgMethod
+ forStBackend.setLocalDbStoragePaths(null);
+ assertNull(forStBackend.getLocalDbStoragePaths());
+ } finally {
+ keyedBackend.dispose();
+ env.close();
+ }
+ }
+
+ /** Validates that empty arguments for the local DB path are invalid. */
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetEmptyPaths() throws Exception {
+ ForStStateBackend forStStateBackend = new ForStStateBackend();
+ forStStateBackend.setLocalDbStoragePaths();
+ }
+
+ /** Validates that schemes other than 'file:/' are not allowed. */
+ @Test(expected = IllegalArgumentException.class)
+ public void testNonFileSchemePath() throws Exception {
+ ForStStateBackend forStStateBackend = new ForStStateBackend();
+ forStStateBackend.setLocalDbStoragePath("hdfs:///some/path/to/perdition");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testDbPathRelativePaths() throws Exception {
+ ForStStateBackend forStStateBackend = new ForStStateBackend();
+ forStStateBackend.setLocalDbStoragePath("relative/path");
+ }
+
+ @Test
+ @Timeout(value = 60)
+ public void testCleanRelocatedDbLogs() throws Exception {
+ final File folder = tempFolder.newFolder();
+ final File relocatedDBLogDir = tempFolder.newFolder("db_logs");
+ final File logFile = new File(relocatedDBLogDir, "taskManager.log");
+ Files.createFile(logFile.toPath());
+ System.setProperty("log.file", logFile.getAbsolutePath());
+
+ Configuration conf = new Configuration();
+ conf.set(ForStConfigurableOptions.LOG_LEVEL, InfoLogLevel.DEBUG_LEVEL);
+ conf.set(ForStConfigurableOptions.LOG_FILE_NUM, 4);
+ conf.set(ForStConfigurableOptions.LOG_MAX_FILE_SIZE, MemorySize.parse("1kb"));
+ final ForStStateBackend rocksDbBackend =
+ new ForStStateBackend().configure(conf, getClass().getClassLoader());
+ final String dbStoragePath = new Path(folder.toURI().toString()).toString();
+ rocksDbBackend.setLocalDbStoragePath(dbStoragePath);
+
+ final MockEnvironment env = getMockEnvironment(tempFolder.newFolder());
+ ForStKeyedStateBackend<Integer> keyedBackend =
+ createKeyedStateBackend(rocksDbBackend, env, IntSerializer.INSTANCE);
+
+ File instanceBasePath = keyedBackend.getInstanceBasePath();
+ File instanceRocksDBPath = new File(instanceBasePath, "db");
+
+ // avoid tests without relocate.
+ Assume.assumeTrue(instanceRocksDBPath.getAbsolutePath().length() <= 255 - "_LOG".length());
+
+ java.nio.file.Path[] relocatedDbLogs;
+ try {
+ relocatedDbLogs = FileUtils.listDirectory(relocatedDBLogDir.toPath());
+ while (relocatedDbLogs.length <= 2) {
+ // If the default number of log files in rocksdb is not enough, add more logs.
+ try (FlushOptions flushOptions = new FlushOptions()) {
+ keyedBackend.db.put(RandomUtils.nextBytes(32), RandomUtils.nextBytes(512));
+ keyedBackend.db.flush(flushOptions);
+ }
+ relocatedDbLogs = FileUtils.listDirectory(relocatedDBLogDir.toPath());
+ }
+ } finally {
+ keyedBackend.dispose();
+ env.close();
+ }
+
+ relocatedDbLogs = FileUtils.listDirectory(relocatedDBLogDir.toPath());
+ assertEquals(1, relocatedDbLogs.length);
+ assertEquals("taskManager.log", relocatedDbLogs[0].toFile().getName());
+ }
+
+ // ------------------------------------------------------------------------
+ // RocksDB local file automatic from temp directories
+ // ------------------------------------------------------------------------
+
+ /**
+ * This tests whether the RocksDB backends uses the temp directories that are provided from the
+ * {@link Environment} when no db storage path is set.
+ */
+ @Test
+ public void testUseTempDirectories() throws Exception {
+ ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+ File dir1 = tempFolder.newFolder();
+
+ assertNull(forStStateBackend.getLocalDbStoragePaths());
+
+ final MockEnvironment env = getMockEnvironment(dir1);
+ JobID jobID = env.getJobID();
+ KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
+ TaskKvStateRegistry kvStateRegistry = env.getTaskKvStateRegistry();
+ CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
+ ForStKeyedStateBackend<Integer> keyedBackend =
+ forStStateBackend.createForStKeyedStateBackend(
+ new KeyedStateBackendParametersImpl<>(
+ env,
+ jobID,
+ "test_op",
+ IntSerializer.INSTANCE,
+ 1,
+ keyGroupRange,
+ kvStateRegistry,
+ TtlTimeProvider.DEFAULT,
+ new UnregisteredMetricsGroup(),
+ Collections.emptyList(),
+ cancelStreamRegistry));
+
+ try {
+ File instanceBasePath = keyedBackend.getInstanceBasePath();
+ assertThat(instanceBasePath.getAbsolutePath(), startsWith(dir1.getAbsolutePath()));
+ } finally {
+ keyedBackend.dispose();
+ env.close();
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // RocksDB local file directory initialization
+ // ------------------------------------------------------------------------
+
+ @Test
+ @Category(FailsInGHAContainerWithRootUser.class)
+ public void testFailWhenNoLocalStorageDir() throws Exception {
+ final File targetDir = tempFolder.newFolder();
+ Assume.assumeTrue(
+ "Cannot mark directory non-writable", targetDir.setWritable(false, false));
+
+ ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+ try (MockEnvironment env = getMockEnvironment(tempFolder.newFolder())) {
+ forStStateBackend.setLocalDbStoragePath(targetDir.getAbsolutePath());
+
+ boolean hasFailure = false;
+ try {
+ JobID jobID = env.getJobID();
+ KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
+ TaskKvStateRegistry kvStateRegistry =
+ new KvStateRegistry().createTaskRegistry(env.getJobID(), new JobVertexID());
+ CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
+ forStStateBackend.createForStKeyedStateBackend(
+ new KeyedStateBackendParametersImpl<>(
+ env,
+ jobID,
+ "foobar",
+ IntSerializer.INSTANCE,
+ 1,
+ keyGroupRange,
+ kvStateRegistry,
+ TtlTimeProvider.DEFAULT,
+ new UnregisteredMetricsGroup(),
+ Collections.emptyList(),
+ cancelStreamRegistry));
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("No local storage directories available"));
+ assertTrue(e.getMessage().contains(targetDir.getAbsolutePath()));
+ hasFailure = true;
+ }
+ assertTrue(
+ "We must see a failure because no storaged directory is feasible.", hasFailure);
+ } finally {
+ //noinspection ResultOfMethodCallIgnored
+ targetDir.setWritable(true, false);
+ }
+ }
+
+ @Test
+ public void testContinueOnSomeDbDirectoriesMissing() throws Exception {
+ final File targetDir1 = tempFolder.newFolder();
+ final File targetDir2 = tempFolder.newFolder();
+ Assume.assumeTrue(
+ "Cannot mark directory non-writable", targetDir1.setWritable(false, false));
+
+ ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+ try (MockEnvironment env = getMockEnvironment(tempFolder.newFolder())) {
+ forStStateBackend.setLocalDbStoragePaths(
+ targetDir1.getAbsolutePath(), targetDir2.getAbsolutePath());
+
+ try {
+ JobID jobID = env.getJobID();
+ KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
+ TaskKvStateRegistry kvStateRegistry =
+ new KvStateRegistry().createTaskRegistry(env.getJobID(), new JobVertexID());
+ CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
+ ForStKeyedStateBackend<Integer> keyedStateBackend =
+ forStStateBackend.createForStKeyedStateBackend(
+ new KeyedStateBackendParametersImpl<>(
+ env,
+ jobID,
+ "foobar",
+ IntSerializer.INSTANCE,
+ 1,
+ keyGroupRange,
+ kvStateRegistry,
+ TtlTimeProvider.DEFAULT,
+ new UnregisteredMetricsGroup(),
+ Collections.emptyList(),
+ cancelStreamRegistry));
+
+ keyedStateBackend.dispose();
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("Backend initialization failed even though some paths were available");
+ }
+ } finally {
+ //noinspection ResultOfMethodCallIgnored
+ targetDir1.setWritable(true, false);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // RocksDB Options
+ // ------------------------------------------------------------------------
+ @Test
+ public void testConfigurableOptionsFromConfig() throws Exception {
+ Configuration configuration = new Configuration();
+
+ // verify illegal configuration
+ {
+ verifyIllegalArgument(ForStConfigurableOptions.MAX_BACKGROUND_THREADS, "-1");
+ verifyIllegalArgument(ForStConfigurableOptions.LOG_LEVEL, "DEBUG");
+ verifyIllegalArgument(ForStConfigurableOptions.LOG_DIR, "tmp/rocksdb-logs/");
+ verifyIllegalArgument(ForStConfigurableOptions.LOG_DIR, "");
+ verifyIllegalArgument(ForStConfigurableOptions.LOG_FILE_NUM, "0");
+ verifyIllegalArgument(ForStConfigurableOptions.LOG_FILE_NUM, "-1");
+ verifyIllegalArgument(ForStConfigurableOptions.LOG_MAX_FILE_SIZE, "-1KB");
+ verifyIllegalArgument(ForStConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "-1");
+ verifyIllegalArgument(ForStConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "-1");
+
+ verifyIllegalArgument(ForStConfigurableOptions.TARGET_FILE_SIZE_BASE, "0KB");
+ verifyIllegalArgument(ForStConfigurableOptions.MAX_SIZE_LEVEL_BASE, "1BB");
+ verifyIllegalArgument(ForStConfigurableOptions.WRITE_BUFFER_SIZE, "-1KB");
+ verifyIllegalArgument(ForStConfigurableOptions.BLOCK_SIZE, "0MB");
+ verifyIllegalArgument(ForStConfigurableOptions.METADATA_BLOCK_SIZE, "0MB");
+ verifyIllegalArgument(ForStConfigurableOptions.BLOCK_CACHE_SIZE, "0");
+
+ verifyIllegalArgument(ForStConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "1");
+
+ verifyIllegalArgument(ForStConfigurableOptions.COMPACTION_STYLE, "LEV");
+ verifyIllegalArgument(ForStConfigurableOptions.COMPRESSION_PER_LEVEL, "SNAP");
+ verifyIllegalArgument(ForStConfigurableOptions.USE_BLOOM_FILTER, "NO");
+ verifyIllegalArgument(ForStConfigurableOptions.BLOOM_FILTER_BLOCK_BASED_MODE, "YES");
+ }
+
+ // verify legal configuration
+ {
+ configuration.setString(ForStConfigurableOptions.LOG_LEVEL.key(), "DEBUG_LEVEL");
+ configuration.setString(ForStConfigurableOptions.LOG_DIR.key(), "/tmp/rocksdb-logs/");
+ configuration.setString(ForStConfigurableOptions.LOG_FILE_NUM.key(), "10");
+ configuration.setString(ForStConfigurableOptions.LOG_MAX_FILE_SIZE.key(), "2MB");
+ configuration.setString(ForStConfigurableOptions.COMPACTION_STYLE.key(), "level");
+ configuration.setString(
+ ForStConfigurableOptions.COMPRESSION_PER_LEVEL.key(),
+ "no_compression;snappy_compression;lz4_compression");
+ configuration.setString(ForStConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE.key(), "TRUE");
+ configuration.setString(ForStConfigurableOptions.TARGET_FILE_SIZE_BASE.key(), "8 mb");
+ configuration.setString(ForStConfigurableOptions.MAX_SIZE_LEVEL_BASE.key(), "128MB");
+ configuration.setString(ForStConfigurableOptions.MAX_BACKGROUND_THREADS.key(), "4");
+ configuration.setString(ForStConfigurableOptions.MAX_WRITE_BUFFER_NUMBER.key(), "4");
+ configuration.setString(
+ ForStConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE.key(), "2");
+ configuration.setString(ForStConfigurableOptions.WRITE_BUFFER_SIZE.key(), "64 MB");
+ configuration.setString(ForStConfigurableOptions.BLOCK_SIZE.key(), "4 kb");
+ configuration.setString(ForStConfigurableOptions.METADATA_BLOCK_SIZE.key(), "8 kb");
+ configuration.setString(ForStConfigurableOptions.BLOCK_CACHE_SIZE.key(), "512 mb");
+ configuration.setString(ForStConfigurableOptions.USE_BLOOM_FILTER.key(), "TRUE");
+
+ try (ForStResourceContainer optionsContainer =
+ new ForStResourceContainer(configuration, null, null, null, false)) {
+
+ DBOptions dbOptions = optionsContainer.getDbOptions();
+ assertEquals(-1, dbOptions.maxOpenFiles());
+ assertEquals(InfoLogLevel.DEBUG_LEVEL, dbOptions.infoLogLevel());
+ assertEquals("/tmp/rocksdb-logs/", dbOptions.dbLogDir());
+ assertEquals(10, dbOptions.keepLogFileNum());
+ assertEquals(2 * SizeUnit.MB, dbOptions.maxLogFileSize());
+
+ ColumnFamilyOptions columnOptions = optionsContainer.getColumnOptions();
+ assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
+ assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
+ assertEquals(8 * SizeUnit.MB, columnOptions.targetFileSizeBase());
+ assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
+ assertEquals(4, columnOptions.maxWriteBufferNumber());
+ assertEquals(2, columnOptions.minWriteBufferNumberToMerge());
+ assertEquals(64 * SizeUnit.MB, columnOptions.writeBufferSize());
+ assertEquals(
+ Arrays.asList(
+ CompressionType.NO_COMPRESSION,
+ CompressionType.SNAPPY_COMPRESSION,
+ CompressionType.LZ4_COMPRESSION),
+ columnOptions.compressionPerLevel());
+
+ BlockBasedTableConfig tableConfig =
+ (BlockBasedTableConfig) columnOptions.tableFormatConfig();
+ assertEquals(4 * SizeUnit.KB, tableConfig.blockSize());
+ assertEquals(8 * SizeUnit.KB, tableConfig.metadataBlockSize());
+ assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
+ assertTrue(tableConfig.filterPolicy() instanceof BloomFilter);
+ }
+ }
+ }
+
+ @Test
+ public void testOptionsFactory() throws Exception {
+ ForStStateBackend forStStateBackend = new ForStStateBackend();
+
+ // verify that user-defined options factory could be configured via config.yaml
+ Configuration config = new Configuration();
+ config.setString(ForStOptions.OPTIONS_FACTORY.key(), TestOptionsFactory.class.getName());
+ config.setString(TestOptionsFactory.BACKGROUND_JOBS_OPTION.key(), "4");
+
+ forStStateBackend = forStStateBackend.configure(config, getClass().getClassLoader());
+
+ assertTrue(forStStateBackend.getForStOptions() instanceof TestOptionsFactory);
+
+ try (ForStResourceContainer optionsContainer =
+ forStStateBackend.createOptionsAndResourceContainer(null)) {
+ DBOptions dbOptions = optionsContainer.getDbOptions();
+ assertEquals(4, dbOptions.maxBackgroundJobs());
+ }
+
+ // verify that user-defined options factory could be set programmatically and override
+ // pre-configured one.
+ forStStateBackend.setForStOptions(
+ new ForStOptionsFactory() {
+ @Override
+ public DBOptions createDBOptions(
+ DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
+ return currentOptions;
+ }
+
+ @Override
+ public ColumnFamilyOptions createColumnOptions(
+ ColumnFamilyOptions currentOptions,
+ Collection<AutoCloseable> handlesToClose) {
+ return currentOptions.setCompactionStyle(CompactionStyle.FIFO);
+ }
+ });
+
+ try (ForStResourceContainer optionsContainer =
+ forStStateBackend.createOptionsAndResourceContainer(null)) {
+ ColumnFamilyOptions colCreated = optionsContainer.getColumnOptions();
+ assertEquals(CompactionStyle.FIFO, colCreated.compactionStyle());
+ }
+ }
+
+ @Test
+ public void testConfigurableOptions() throws Exception {
+ Configuration configuration = new Configuration();
+ configuration.set(ForStConfigurableOptions.COMPACTION_STYLE, CompactionStyle.UNIVERSAL);
+ try (final ForStResourceContainer optionsContainer =
+ new ForStResourceContainer(configuration, null, null, null, false)) {
+
+ final ColumnFamilyOptions columnFamilyOptions = optionsContainer.getColumnOptions();
+ assertNotNull(columnFamilyOptions);
+ assertEquals(CompactionStyle.UNIVERSAL, columnFamilyOptions.compactionStyle());
+ }
+
+ try (final ForStResourceContainer optionsContainer =
+ new ForStResourceContainer(new Configuration(), null, null, null, false)) {
+
+ final ColumnFamilyOptions columnFamilyOptions = optionsContainer.getColumnOptions();
+ assertNotNull(columnFamilyOptions);
+ assertEquals(CompactionStyle.LEVEL, columnFamilyOptions.compactionStyle());
+ }
+ }
+
+ @Test
+ public void testPredefinedAndOptionsFactory() throws Exception {
+ final ForStOptionsFactory optionsFactory =
+ new ForStOptionsFactory() {
+ @Override
+ public DBOptions createDBOptions(
+ DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
+ return currentOptions;
+ }
+
+ @Override
+ public ColumnFamilyOptions createColumnOptions(
+ ColumnFamilyOptions currentOptions,
+ Collection<AutoCloseable> handlesToClose) {
+ return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
+ }
+ };
+
+ try (final ForStResourceContainer optionsContainer =
+ new ForStResourceContainer(optionsFactory)) {
+
+ final ColumnFamilyOptions columnFamilyOptions = optionsContainer.getColumnOptions();
+ assertNotNull(columnFamilyOptions);
+ assertEquals(CompactionStyle.UNIVERSAL, columnFamilyOptions.compactionStyle());
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Reconfiguration
+ // ------------------------------------------------------------------------
+
+ @Test
+ public void testForStReconfigurationCopiesExistingValues() throws Exception {
+ final ForStStateBackend original = new ForStStateBackend();
+ final ForStOptionsFactory optionsFactory = new TestOptionsFactory();
+ original.setForStOptions(optionsFactory);
+
+ final String[] localDirs =
+ new String[] {
+ tempFolder.newFolder().getAbsolutePath(),
+ tempFolder.newFolder().getAbsolutePath()
+ };
+ original.setLocalDbStoragePaths(localDirs);
+
+ ForStStateBackend copy =
+ original.configure(
+ new Configuration(), Thread.currentThread().getContextClassLoader());
+
+ assertArrayEquals(original.getLocalDbStoragePaths(), copy.getLocalDbStoragePaths());
+ assertEquals(original.getForStOptions(), copy.getForStOptions());
+ }
+
+ // ------------------------------------------------------------------------
+ // RocksDB Memory Control
+ // ------------------------------------------------------------------------
+
+ @Test
+ public void testDefaultMemoryControlParameters() {
+ ForStMemoryConfiguration memSettings = new ForStMemoryConfiguration();
+ assertTrue(memSettings.isUsingManagedMemory());
+ assertFalse(memSettings.isUsingFixedMemoryPerSlot());
+ assertEquals(
+ ForStOptions.HIGH_PRIORITY_POOL_RATIO.defaultValue(),
+ memSettings.getHighPriorityPoolRatio(),
+ 0.0);
+ assertEquals(
+ ForStOptions.WRITE_BUFFER_RATIO.defaultValue(),
+ memSettings.getWriteBufferRatio(),
+ 0.0);
+
+ ForStMemoryConfiguration configured =
+ ForStMemoryConfiguration.fromOtherAndConfiguration(
+ memSettings, new Configuration());
+ assertTrue(configured.isUsingManagedMemory());
+ assertFalse(configured.isUsingFixedMemoryPerSlot());
+ assertEquals(
+ ForStOptions.HIGH_PRIORITY_POOL_RATIO.defaultValue(),
+ configured.getHighPriorityPoolRatio(),
+ 0.0);
+ assertEquals(
+ ForStOptions.WRITE_BUFFER_RATIO.defaultValue(),
+ configured.getWriteBufferRatio(),
+ 0.0);
+ }
+
+ @Test
+ public void testConfigureManagedMemory() {
+ final Configuration config = new Configuration();
+ config.set(ForStOptions.USE_MANAGED_MEMORY, true);
+
+ final ForStMemoryConfiguration memSettings =
+ ForStMemoryConfiguration.fromOtherAndConfiguration(
+ new ForStMemoryConfiguration(), config);
+
+ assertTrue(memSettings.isUsingManagedMemory());
+ }
+
+ @Test
+ public void testConfigureIllegalMemoryControlParameters() {
+ ForStMemoryConfiguration memSettings = new ForStMemoryConfiguration();
+
+ verifySetParameter(() -> memSettings.setFixedMemoryPerSlot("-1B"));
+ verifySetParameter(() -> memSettings.setHighPriorityPoolRatio(-0.1));
+ verifySetParameter(() -> memSettings.setHighPriorityPoolRatio(1.1));
+ verifySetParameter(() -> memSettings.setWriteBufferRatio(-0.1));
+ verifySetParameter(() -> memSettings.setWriteBufferRatio(1.1));
+
+ memSettings.setFixedMemoryPerSlot("128MB");
+ memSettings.setWriteBufferRatio(0.6);
+ memSettings.setHighPriorityPoolRatio(0.6);
+
+ try {
+ // sum of writeBufferRatio and highPriPoolRatio larger than 1.0
+ memSettings.validate();
+ fail("Expected an IllegalArgumentException.");
+ } catch (IllegalArgumentException expected) {
+ // expected exception
+ }
+ }
+
+ private void verifySetParameter(Runnable setter) {
+ try {
+ setter.run();
+ fail("No expected IllegalArgumentException.");
+ } catch (IllegalArgumentException expected) {
+ // expected exception
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Utilities
+ // ------------------------------------------------------------------------
+
+ static MockEnvironment getMockEnvironment(File tempDir) {
+ return MockEnvironment.builder()
+ .setUserCodeClassLoader(ForStStateBackendConfigTest.class.getClassLoader())
+ .setTaskManagerRuntimeInfo(
+ new TestingTaskManagerRuntimeInfo(new Configuration(), tempDir))
+ .build();
+ }
+
+ private void verifyIllegalArgument(ConfigOption<?> configOption, String configValue) {
+ Configuration configuration = new Configuration();
+ configuration.setString(configOption.key(), configValue);
+
+ ForStStateBackend stateBackend = new ForStStateBackend();
+ try {
+ stateBackend.configure(configuration, null);
+ fail("Not throwing expected IllegalArgumentException.");
+ } catch (IllegalArgumentException e) {
+ // ignored
+ }
+ }
+
+ /** An implementation of options factory for testing. */
+ public static class TestOptionsFactory implements ConfigurableForStOptionsFactory {
+ public static final ConfigOption<Integer> BACKGROUND_JOBS_OPTION =
+ ConfigOptions.key("my.custom.rocksdb.backgroundJobs").intType().defaultValue(2);
+
+ private int backgroundJobs = BACKGROUND_JOBS_OPTION.defaultValue();
+
+ @Override
+ public DBOptions createDBOptions(
+ DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
+ return currentOptions.setMaxBackgroundJobs(backgroundJobs);
+ }
+
+ @Override
+ public ColumnFamilyOptions createColumnOptions(
+ ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
+ return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
+ }
+
+ @Override
+ public ForStOptionsFactory configure(ReadableConfig configuration) {
+ this.backgroundJobs = configuration.get(BACKGROUND_JOBS_OPTION);
+ return this;
+ }
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendFactoryTest.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendFactoryTest.java
new file mode 100644
index 00000000000..effcd2be793
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStStateBackendFactoryTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.configuration.CheckpointingOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.StateBackendOptions;
+import org.apache.flink.runtime.state.StateBackend;
+import org.apache.flink.runtime.state.StateBackendLoader;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashSet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/** Tests for the ForStStateBackendFactory. */
+public class ForStStateBackendFactoryTest {
+
+ @Rule public final TemporaryFolder tmp = new TemporaryFolder();
+
+ private final ClassLoader cl = getClass().getClassLoader();
+
+ private final String backendKey = StateBackendOptions.STATE_BACKEND.key();
+
+ // ------------------------------------------------------------------------
+
+ @Test
+ public void testEmbeddedFactoryName() {
+ // construct the name such that it will not be automatically adjusted on refactorings
+ String factoryName = "org.apache.flink.state.forst.For";
+ factoryName += "StStateBackendFactory";
+
+ // !!! if this fails, the code in StateBackendLoader must be adjusted
+ assertEquals(factoryName, ForStStateBackendFactory.class.getName());
+ }
+
+ /**
+ * Validates loading a file system state backend with additional parameters from the cluster
+ * configuration.
+ */
+ @Test
+ public void testLoadForStStateBackend() throws Exception {
+ final String localDir1 = tmp.newFolder().getAbsolutePath();
+ final String localDir2 = tmp.newFolder().getAbsolutePath();
+ final String localDirs = localDir1 + File.pathSeparator + localDir2;
+ final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();
+
+ // TODO: Support short name of backendKey
+
+ final Configuration config1 = new Configuration();
+ config1.setString(backendKey, ForStStateBackendFactory.class.getName());
+ config1.set(ForStOptions.LOCAL_DIRECTORIES, localDirs);
+ config1.set(CheckpointingOptions.INCREMENTAL_CHECKPOINTS, incremental);
+
+ StateBackend backend1 = StateBackendLoader.loadStateBackendFromConfig(config1, cl, null);
+
+ assertTrue(backend1 instanceof ForStStateBackend);
+
+ ForStStateBackend fs1 = (ForStStateBackend) backend1;
+
+ checkPaths(fs1.getLocalDbStoragePaths(), localDir1, localDir2);
+ }
+
+ /**
+ * Validates taking the application-defined ForSt state backend and adding with additional
+ * parameters from the cluster configuration, but giving precedence to application-defined
+ * parameters over configuration-defined parameters.
+ */
+ @Test
+ public void testLoadForStStateBackendMixed() throws Exception {
+ final String localDir1 = tmp.newFolder().getAbsolutePath();
+ final String localDir2 = tmp.newFolder().getAbsolutePath();
+ final String localDir3 = tmp.newFolder().getAbsolutePath();
+ final String localDir4 = tmp.newFolder().getAbsolutePath();
+
+ final ForStStateBackend backend = new ForStStateBackend();
+ backend.setLocalDbStoragePaths(localDir1, localDir2);
+
+ final Configuration config = new Configuration();
+ config.setString(backendKey, "hashmap"); // this should not be picked up
+ config.set(
+ ForStOptions.LOCAL_DIRECTORIES,
+ localDir3 + ":" + localDir4); // this should not be picked up
+
+ final StateBackend loadedBackend =
+ StateBackendLoader.fromApplicationOrConfigOrDefault(
+ backend, new Configuration(), config, cl, null);
+ assertTrue(loadedBackend instanceof ForStStateBackend);
+
+ final ForStStateBackend loadedRocks = (ForStStateBackend) loadedBackend;
+
+ checkPaths(loadedRocks.getLocalDbStoragePaths(), localDir1, localDir2);
+ }
+
+ // ------------------------------------------------------------------------
+
+ private static void checkPaths(String[] pathsArray, String... paths) {
+ assertNotNull(pathsArray);
+ assertNotNull(paths);
+
+ assertEquals(pathsArray.length, paths.length);
+
+ HashSet<String> pathsSet = new HashSet<>(Arrays.asList(pathsArray));
+
+ for (String path : paths) {
+ assertTrue(pathsSet.contains(path));
+ }
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStTestUtils.java b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStTestUtils.java
new file mode 100644
index 00000000000..e5cb4c91872
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-forst/src/test/java/org/apache/flink/state/forst/ForStTestUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.state.forst;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateBackendParametersImpl;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+
+import java.io.IOException;
+import java.util.Collections;
+
+/** Test utils for the ForSt state backend. */
+public final class ForStTestUtils {
+ public static <K> ForStKeyedStateBackend<K> createKeyedStateBackend(
+ ForStStateBackend forStStateBackend, Environment env, TypeSerializer<K> keySerializer)
+ throws IOException {
+
+ return forStStateBackend.createForStKeyedStateBackend(
+ new KeyedStateBackendParametersImpl<>(
+ env,
+ env.getJobID(),
+ "test_op",
+ keySerializer,
+ 1,
+ new KeyGroupRange(0, 0),
+ env.getTaskKvStateRegistry(),
+ TtlTimeProvider.DEFAULT,
+ new UnregisteredMetricsGroup(),
+ (name, value) -> {},
+ Collections.emptyList(),
+ new CloseableRegistry(),
+ 1.0));
+ }
+}