You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by wa...@apache.org on 2022/04/25 02:42:10 UTC
[flink-kubernetes-operator] branch main updated: [FLINK-27334] Support auto generate the doc for the KubernetesOperatorConfigOptions
This is an automated email from the ASF dual-hosted git repository.
wangyang0918 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 5a81a09 [FLINK-27334] Support auto generate the doc for the KubernetesOperatorConfigOptions
5a81a09 is described below
commit 5a81a09bd664728449cdb2574853a28309e3465e
Author: SteNicholas <pr...@163.com>
AuthorDate: Mon Apr 25 09:43:50 2022 +0800
[FLINK-27334] Support auto generate the doc for the KubernetesOperatorConfigOptions
---
Dockerfile | 4 +-
docs/content/docs/operations/configuration.md | 15 +-
.../kubernetes_operator_config_configuration.html | 84 +++
flink-kubernetes-docs/README.md | 35 ++
flink-kubernetes-docs/pom.xml | 156 +++++
.../configuration/ConfigOptionsDocGenerator.java | 629 +++++++++++++++++++++
.../docs/configuration/OptionsClassLocation.java | 45 ++
.../flink/kubernetes/operator/docs/util/Utils.java | 36 ++
.../src/main/resources/log4j.properties | 24 +
.../ConfigOptionsDocGeneratorTest.java | 607 ++++++++++++++++++++
.../ConfigOptionsDocsCompletenessITCase.java | 377 ++++++++++++
.../docs/configuration/data/TestCommonOptions.java | 51 ++
flink-kubernetes-webhook/pom.xml | 4 -
pom.xml | 26 +
14 files changed, 2074 insertions(+), 19 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index b45eb23..85eed80 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,6 +23,7 @@ WORKDIR /app
ENV SHADED_DIR=flink-kubernetes-shaded
ENV OPERATOR_DIR=flink-kubernetes-operator
ENV WEBHOOK_DIR=flink-kubernetes-webhook
+ENV DOCS_DIR=flink-kubernetes-docs
RUN mkdir $OPERATOR_DIR $WEBHOOK_DIR
@@ -30,6 +31,7 @@ COPY pom.xml .
COPY $SHADED_DIR/pom.xml ./$SHADED_DIR/
COPY $WEBHOOK_DIR/pom.xml ./$WEBHOOK_DIR/
COPY $OPERATOR_DIR/pom.xml ./$OPERATOR_DIR/
+COPY $DOCS_DIR/pom.xml ./$DOCS_DIR/
COPY $OPERATOR_DIR/src ./$OPERATOR_DIR/src
COPY $WEBHOOK_DIR/src ./$WEBHOOK_DIR/src
@@ -37,7 +39,7 @@ COPY $WEBHOOK_DIR/src ./$WEBHOOK_DIR/src
COPY .git ./.git
COPY tools ./tools
-RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -DskipTests=$SKIP_TESTS
+RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -pl !flink-kubernetes-docs -DskipTests=$SKIP_TESTS
# stage
FROM openjdk:11-jre
diff --git a/docs/content/docs/operations/configuration.md b/docs/content/docs/operations/configuration.md
index 6b31350..19d7206 100644
--- a/docs/content/docs/operations/configuration.md
+++ b/docs/content/docs/operations/configuration.md
@@ -52,17 +52,4 @@ To learn more about metrics and logging configuration please refer to the dedica
## Operator Configuration Reference
-| Key | Default | Type | Description |
-| ---- | ------- | ---- | ----------- |
-| kubernetes.operator.reconciler.reschedule.interval | 60s | Duration | The interval for the controller to reschedule the reconcile process. |
-| kubernetes.operator.observer.rest-ready.delay | 10s | Duration | Final delay before deployment is marked ready after port becomes accessible. |
-| kubernetes.operator.reconciler.max.parallelism | 5 | Integer | The maximum number of threads running the reconciliation loop. Use -1 for infinite. |
-| kubernetes.operator.observer.progress-check.interval | 10s | Duration | The interval for observing status for in-progress operations such as deployment and savepoints. |
-| kubernetes.operator.observer.savepoint.trigger.grace-period | 10s | Duration | The interval before a savepoint trigger attempt is marked as unsuccessful. |
-| kubernetes.operator.observer.flink.client.timeout | 10s | Duration | The timeout for the observer to wait the flink rest client to return. |
-| kubernetes.operator.reconciler.flink.cancel.job.timeout | 1min | Duration | The timeout for the reconciler to wait for flink to cancel job. |
-| kubernetes.operator.reconciler.flink.cluster.shutdown.timeout | 60s | Duration | The timeout for the reconciler to wait for flink to shutdown cluster. |
-| kubernetes.operator.deployment.rollback.enabled | false | Boolean | Whether to enable rolling back failed deployment upgrades. |
-| kubernetes.operator.deployment.readiness.timeout | 1min | Duration | The timeout for deployments to become ready/stable before being rolled back if rollback is enabled. |
-| kubernetes.operator.user.artifacts.base.dir | /opt/flink/artifacts | String | The base dir to put the session job artifacts. |
-| kubernetes.operator.job.upgrade.ignore-pending-savepoint | false | Boolean | Whether to ignore pending savepoint during job upgrade. |
+{{< generated/kubernetes_operator_config_configuration >}}
diff --git a/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html b/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html
new file mode 100644
index 0000000..2923c8a
--- /dev/null
+++ b/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html
@@ -0,0 +1,84 @@
+<table class="configuration table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>kubernetes.operator.deployment.readiness.timeout</h5></td>
+ <td style="word-wrap: break-word;">1 min</td>
+ <td>Duration</td>
+ <td>The timeout for deployments to become ready/stable before being rolled back if rollback is enabled.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.deployment.rollback.enabled</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to enable rolling back failed deployment upgrades.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.job.upgrade.ignore-pending-savepoint</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to ignore pending savepoint during job upgrade.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.observer.flink.client.timeout</h5></td>
+ <td style="word-wrap: break-word;">10 s</td>
+ <td>Duration</td>
+ <td>The timeout for the observer to wait the flink rest client to return.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.observer.progress-check.interval</h5></td>
+ <td style="word-wrap: break-word;">10 s</td>
+ <td>Duration</td>
+ <td>The interval for observing status for in-progress operations such as deployment and savepoints.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.observer.rest-ready.delay</h5></td>
+ <td style="word-wrap: break-word;">10 s</td>
+ <td>Duration</td>
+ <td>Final delay before deployment is marked ready after port becomes accessible.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.observer.savepoint.trigger.grace-period</h5></td>
+ <td style="word-wrap: break-word;">10 s</td>
+ <td>Duration</td>
+ <td>The interval before a savepoint trigger attempt is marked as unsuccessful.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.reconciler.flink.cancel.job.timeout</h5></td>
+ <td style="word-wrap: break-word;">1 min</td>
+ <td>Duration</td>
+ <td>The timeout for the reconciler to wait for flink to cancel job.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.reconciler.flink.cluster.shutdown.timeout</h5></td>
+ <td style="word-wrap: break-word;">1 min</td>
+ <td>Duration</td>
+ <td>The timeout for the reconciler to wait for flink to shutdown cluster.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.reconciler.max.parallelism</h5></td>
+ <td style="word-wrap: break-word;">5</td>
+ <td>Integer</td>
+ <td>The maximum number of threads running the reconciliation loop. Use -1 for infinite.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.reconciler.reschedule.interval</h5></td>
+ <td style="word-wrap: break-word;">1 min</td>
+ <td>Duration</td>
+ <td>The interval for the controller to reschedule the reconcile process.</td>
+ </tr>
+ <tr>
+ <td><h5>kubernetes.operator.user.artifacts.base.dir</h5></td>
+ <td style="word-wrap: break-word;">"/opt/flink/artifacts"</td>
+ <td>String</td>
+ <td>The base dir to put the session job artifacts.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/flink-kubernetes-docs/README.md b/flink-kubernetes-docs/README.md
new file mode 100644
index 0000000..9a04abb
--- /dev/null
+++ b/flink-kubernetes-docs/README.md
@@ -0,0 +1,35 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Documentation generators
+
+This module contains generators that create HTML files directly from Flink Kubernetes Operator's source code.
+
+## Configuration documentation
+
+The `ConfigOptionsDocGenerator` can be used to generate a reference of `ConfigOptions`. By default, a separate file is generated for each `*Options` class found in `org.apache.flink.kubernetes.operator.docs.configuration`. The `@ConfigGroups` annotation can be used to generate multiple files from a single class.
+
+To integrate an `*Options` class from another package, add another module-package argument pair to `ConfigOptionsDocGenerator#LOCATIONS`.
+
+The files can be generated by running `mvn package -Pgenerate-docs -pl flink-kubernetes-docs -nsu -DskipTests`, and can be integrated into the documentation using `{{ include generated/<file-name> >}}`.
+
+The documentation must be regenerated whenever
+* an `*Options` class was added or removed
+* a `ConfigOption` was added to or removed from an `*Options` class
+* a `ConfigOption` was modified in any way.
diff --git a/flink-kubernetes-docs/pom.xml b/flink-kubernetes-docs/pom.xml
new file mode 100644
index 0000000..07bd22a
--- /dev/null
+++ b/flink-kubernetes-docs/pom.xml
@@ -0,0 +1,156 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-kubernetes-operator-parent</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-kubernetes-docs</artifactId>
+ <name>Flink Kubernetes Docs</name>
+
+ <properties>
+ <generated.docs.dir>./docs/layouts/shortcodes/generated</generated.docs.dir>
+ <assertj.version>3.21.0</assertj.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-kubernetes-operator</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Test -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <version>${junit.jupiter.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <version>${assertj.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <!-- Used for parsing HTML -->
+ <groupId>org.jsoup</groupId>
+ <artifactId>jsoup</artifactId>
+ <version>1.14.2</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <!-- Not important for this module as we only access Flink classes-->
+ <id>dependency-convergence</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ <rules>
+ <dependencyConvergence/>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemPropertyVariables>
+ <rootDir>${rootDir}</rootDir>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>generate-docs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-docs</id>
+ <phase>package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <mkdir dir="${rootDir}/${generated.docs.dir}"/>
+ <java classname="org.apache.flink.kubernetes.operator.docs.configuration.ConfigOptionsDocGenerator"
+ fork="true" failonerror="true">
+ <classpath refid="maven.compile.classpath"/>
+ <arg value="${rootDir}/${generated.docs.dir}/"/>
+ <arg value="${rootDir}"/>
+ </java>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git a/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGenerator.java b/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGenerator.java
new file mode 100644
index 0000000..ac1dc2f
--- /dev/null
+++ b/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGenerator.java
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.docs.configuration;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.annotation.docs.ConfigGroup;
+import org.apache.flink.annotation.docs.ConfigGroups;
+import org.apache.flink.annotation.docs.Documentation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.DescribedEnum;
+import org.apache.flink.configuration.description.Description;
+import org.apache.flink.configuration.description.Formatter;
+import org.apache.flink.configuration.description.HtmlFormatter;
+import org.apache.flink.configuration.description.InlineElement;
+import org.apache.flink.configuration.description.TextElement;
+import org.apache.flink.util.TimeUtils;
+import org.apache.flink.util.function.ThrowingConsumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.configuration.description.TextElement.text;
+import static org.apache.flink.kubernetes.operator.docs.util.Utils.escapeCharacters;
+
+/** Class used for generating code based documentation of configuration parameters. */
+public class ConfigOptionsDocGenerator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ConfigOptionsDocGenerator.class);
+
+ static final OptionsClassLocation[] LOCATIONS =
+ new OptionsClassLocation[] {
+ new OptionsClassLocation(
+ "flink-kubernetes-operator", "org.apache.flink.kubernetes.operator.config")
+ };
+ static final String DEFAULT_PATH_PREFIX = "src/main/java";
+
+ private static final String CLASS_NAME_GROUP = "className";
+ private static final String CLASS_PREFIX_GROUP = "classPrefix";
+ private static final Pattern CLASS_NAME_PATTERN =
+ Pattern.compile(
+ "(?<"
+ + CLASS_NAME_GROUP
+ + ">(?<"
+ + CLASS_PREFIX_GROUP
+ + ">[a-zA-Z]*)(?:Options|Config|Parameters))(?:\\.java)?");
+
+ private static final Formatter formatter = new HtmlFormatter();
+
+ /**
+ * This method generates html tables from set of classes containing {@link ConfigOption
+ * ConfigOptions}.
+ *
+ * <p>For each class 1 or more html tables will be generated and placed into a separate file,
+ * depending on whether the class is annotated with {@link ConfigGroups}. The tables contain the
+ * key, default value and description for every {@link ConfigOption}.
+ *
+ * <p>One additional table is generated containing all {@link ConfigOption ConfigOptions} that
+ * are annotated with {@link Documentation.Section}.
+ *
+ * @param args [0] output directory for the generated files [1] project root directory
+ */
+ public static void main(String[] args) throws IOException, ClassNotFoundException {
+ String outputDirectory = args[0];
+ String rootDir = args[1];
+ LOG.info(
+ "Searching the following locations; configured via {}#LOCATIONS:{}",
+ ConfigOptionsDocGenerator.class.getCanonicalName(),
+ Arrays.stream(LOCATIONS)
+ .map(OptionsClassLocation::toString)
+ .collect(Collectors.joining("\n\t", "\n\t", "")));
+ for (OptionsClassLocation location : LOCATIONS) {
+ createTable(rootDir, location.getModule(), location.getPackage(), outputDirectory);
+ }
+
+ generateCommonSection(rootDir, outputDirectory, LOCATIONS, DEFAULT_PATH_PREFIX);
+ }
+
+ @VisibleForTesting
+ static void generateCommonSection(
+ String rootDir,
+ String outputDirectory,
+ OptionsClassLocation[] locations,
+ String pathPrefix)
+ throws IOException, ClassNotFoundException {
+ List<OptionWithMetaInfo> allSectionOptions = new ArrayList<>(32);
+ for (OptionsClassLocation location : locations) {
+ allSectionOptions.addAll(
+ findSectionOptions(
+ rootDir, location.getModule(), location.getPackage(), pathPrefix));
+ }
+
+ Map<String, List<OptionWithMetaInfo>> optionsGroupedBySection =
+ allSectionOptions.stream()
+ .flatMap(
+ option -> {
+ final String[] sections =
+ option.field
+ .getAnnotation(Documentation.Section.class)
+ .value();
+ if (sections.length == 0) {
+ throw new RuntimeException(
+ String.format(
+ "Option %s is annotated with %s but the list of sections is empty.",
+ option.option.key(),
+ Documentation.Section.class
+ .getSimpleName()));
+ }
+
+ return Arrays.stream(sections)
+ .map(section -> Tuple2.of(section, option));
+ })
+ .collect(
+ Collectors.groupingBy(
+ option -> option.f0,
+ Collectors.mapping(
+ option -> option.f1, Collectors.toList())));
+
+ optionsGroupedBySection.forEach(
+ (section, options) -> {
+ options.sort(
+ (o1, o2) -> {
+ int position1 =
+ o1.field
+ .getAnnotation(Documentation.Section.class)
+ .position();
+ int position2 =
+ o2.field
+ .getAnnotation(Documentation.Section.class)
+ .position();
+ if (position1 == position2) {
+ return o1.option.key().compareTo(o2.option.key());
+ } else {
+ return Integer.compare(position1, position2);
+ }
+ });
+
+ String sectionHtmlTable = toHtmlTable(options);
+ try {
+ Files.write(
+ Paths.get(outputDirectory, getSectionFileName(section)),
+ sectionHtmlTable.getBytes(StandardCharsets.UTF_8));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @VisibleForTesting
+ static String getSectionFileName(String section) {
+ return section + "_section.html";
+ }
+
+ private static Collection<OptionWithMetaInfo> findSectionOptions(
+ String rootDir, String module, String packageName, String pathPrefix)
+ throws IOException, ClassNotFoundException {
+ Collection<OptionWithMetaInfo> commonOptions = new ArrayList<>(32);
+ processConfigOptions(
+ rootDir,
+ module,
+ packageName,
+ pathPrefix,
+ optionsClass ->
+ extractConfigOptions(optionsClass).stream()
+ .filter(
+ optionWithMetaInfo ->
+ optionWithMetaInfo.field.getAnnotation(
+ Documentation.Section.class)
+ != null)
+ .forEachOrdered(commonOptions::add));
+ return commonOptions;
+ }
+
+ private static void createTable(
+ String rootDir, String module, String packageName, String outputDirectory)
+ throws IOException, ClassNotFoundException {
+ processConfigOptions(
+ rootDir,
+ module,
+ packageName,
+ ConfigOptionsDocGenerator.DEFAULT_PATH_PREFIX,
+ optionsClass -> {
+ List<Tuple2<ConfigGroup, String>> tables = generateTablesForClass(optionsClass);
+ for (Tuple2<ConfigGroup, String> group : tables) {
+ String name;
+ if (group.f0 == null) {
+ Matcher matcher =
+ CLASS_NAME_PATTERN.matcher(optionsClass.getSimpleName());
+ if (!matcher.matches()) {
+ throw new RuntimeException(
+ "Pattern did not match for "
+ + optionsClass.getSimpleName()
+ + '.');
+ }
+ name = matcher.group(CLASS_PREFIX_GROUP);
+ } else {
+ name = group.f0.name();
+ }
+ String outputFile = toSnakeCase(name) + "_configuration.html";
+ Files.write(
+ Paths.get(outputDirectory, outputFile),
+ group.f1.getBytes(StandardCharsets.UTF_8));
+ }
+ });
+ }
+
+ @VisibleForTesting
+ static String toSnakeCase(String name) {
+ return name.replaceAll("(.)([A-Z][a-z])", "$1_$2").toLowerCase();
+ }
+
+ @VisibleForTesting
+ static void processConfigOptions(
+ String rootDir,
+ String module,
+ String packageName,
+ String pathPrefix,
+ ThrowingConsumer<Class<?>, IOException> classConsumer)
+ throws IOException, ClassNotFoundException {
+ Path configDir = Paths.get(rootDir, module, pathPrefix, packageName.replaceAll("\\.", "/"));
+ try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir)) {
+ for (Path entry : stream) {
+ String fileName = entry.getFileName().toString();
+ Matcher matcher = CLASS_NAME_PATTERN.matcher(fileName);
+ if (matcher.matches()) {
+ classConsumer.accept(
+ Class.forName(packageName + '.' + matcher.group(CLASS_NAME_GROUP)));
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ static List<Tuple2<ConfigGroup, String>> generateTablesForClass(Class<?> optionsClass) {
+ ConfigGroups configGroups = optionsClass.getAnnotation(ConfigGroups.class);
+ List<OptionWithMetaInfo> allOptions = extractConfigOptions(optionsClass);
+ if (allOptions.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List<Tuple2<ConfigGroup, String>> tables;
+ if (configGroups != null) {
+ tables = new ArrayList<>(configGroups.groups().length + 1);
+ Tree tree = new Tree(configGroups.groups(), allOptions);
+ for (ConfigGroup group : configGroups.groups()) {
+ List<OptionWithMetaInfo> configOptions = tree.findConfigOptions(group);
+ if (!configOptions.isEmpty()) {
+ sortOptions(configOptions);
+ tables.add(Tuple2.of(group, toHtmlTable(configOptions)));
+ }
+ }
+ List<OptionWithMetaInfo> configOptions = tree.getDefaultOptions();
+ if (!configOptions.isEmpty()) {
+ sortOptions(configOptions);
+ tables.add(Tuple2.of(null, toHtmlTable(configOptions)));
+ }
+ } else {
+ sortOptions(allOptions);
+ tables = Collections.singletonList(Tuple2.of(null, toHtmlTable(allOptions)));
+ }
+ return tables;
+ }
+
+ @VisibleForTesting
+ static List<OptionWithMetaInfo> extractConfigOptions(Class<?> clazz) {
+ try {
+ List<OptionWithMetaInfo> configOptions = new ArrayList<>(8);
+ Field[] fields = clazz.getFields();
+ for (Field field : fields) {
+ if (isConfigOption(field) && shouldBeDocumented(field)) {
+ configOptions.add(
+ new OptionWithMetaInfo((ConfigOption<?>) field.get(null), field));
+ }
+ }
+ return configOptions;
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Failed to extract config options from class " + clazz + '.', e);
+ }
+ }
+
+ private static boolean isConfigOption(Field field) {
+ return field.getType().equals(ConfigOption.class);
+ }
+
+ private static boolean shouldBeDocumented(Field field) {
+ return field.getAnnotation(Deprecated.class) == null
+ && field.getAnnotation(Documentation.ExcludeFromDocumentation.class) == null;
+ }
+
+ /**
+ * Transforms this configuration group into HTML formatted table. Options are sorted
+ * alphabetically by key.
+ *
+ * @param options list of options to include in this group
+ * @return string containing HTML formatted table
+ */
+ private static String toHtmlTable(final List<OptionWithMetaInfo> options) {
+ StringBuilder htmlTable = new StringBuilder();
+ htmlTable.append("<table class=\"configuration table table-bordered\">\n");
+ htmlTable.append(" <thead>\n");
+ htmlTable.append(" <tr>\n");
+ htmlTable.append(" <th class=\"text-left\" style=\"width: 20%\">Key</th>\n");
+ htmlTable.append(" <th class=\"text-left\" style=\"width: 15%\">Default</th>\n");
+ htmlTable.append(" <th class=\"text-left\" style=\"width: 10%\">Type</th>\n");
+ htmlTable.append(
+ " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n");
+ htmlTable.append(" </tr>\n");
+ htmlTable.append(" </thead>\n");
+ htmlTable.append(" <tbody>\n");
+ for (OptionWithMetaInfo option : options) {
+ htmlTable.append(toHtmlString(option));
+ }
+ htmlTable.append(" </tbody>\n");
+ htmlTable.append("</table>\n");
+ return htmlTable.toString();
+ }
+
+ /**
+ * Transforms option to table row.
+ *
+ * @param optionWithMetaInfo option to transform
+ * @return row with the option description
+ */
+ private static String toHtmlString(final OptionWithMetaInfo optionWithMetaInfo) {
+ return ""
+ + " <tr>\n"
+ + " <td><h5>"
+ + escapeCharacters(optionWithMetaInfo.option.key())
+ + "</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">"
+ + escapeCharacters(addWordBreakOpportunities(stringifyDefault(optionWithMetaInfo)))
+ + "</td>\n"
+ + " <td>"
+ + typeToHtml(optionWithMetaInfo)
+ + "</td>\n"
+ + " <td>"
+ + getDescription(optionWithMetaInfo)
+ + "</td>\n"
+ + " </tr>\n";
+ }
+
+ @VisibleForTesting
+ static String getDescription(OptionWithMetaInfo optionWithMetaInfo) {
+ return formatter.format(optionWithMetaInfo.option.description())
+ + Optional.ofNullable(getEnumOptionsDescription(optionWithMetaInfo))
+ .map(formatter::format)
+ .map(desc -> String.format("<br /><br />%s", desc))
+ .orElse("");
+ }
+
+ /**
+ * Returns a {@link Description} for the enum constants of the given option in case it is
+ * enum-based, and {@code null} otherwise.
+ */
+ private static @Nullable Description getEnumOptionsDescription(
+ OptionWithMetaInfo optionWithMetaInfo) {
+ Class<?> clazz = getClazz(optionWithMetaInfo.option);
+ if (!clazz.isEnum()) {
+ return null;
+ }
+ InlineElement[] optionDescriptions =
+ Arrays.stream(clazz.getEnumConstants())
+ .map(ConfigOptionsDocGenerator::formatEnumOption)
+ .map(elements -> TextElement.wrap(elements.toArray(new InlineElement[0])))
+ .toArray(InlineElement[]::new);
+ return Description.builder().text("Possible values:").list(optionDescriptions).build();
+ }
+
+ /**
+ * Formats a single enum constant.
+ *
+ * <p>If the enum implements {@link DescribedEnum}, this includes the given description for each
+ * constant. Otherwise, only the constant itself is printed.
+ */
+ private static List<InlineElement> formatEnumOption(Object e) {
+ final List<InlineElement> elements = new LinkedList<>();
+ elements.add(text("\"%s\"", text(escapeCharacters(e.toString()))));
+
+ if (DescribedEnum.class.isAssignableFrom(e.getClass())) {
+ elements.add(text(": "));
+ elements.add(((DescribedEnum) e).getDescription());
+ }
+
+ return elements;
+ }
+
+ private static Class<?> getClazz(ConfigOption<?> option) {
+ try {
+ Method getClazzMethod = ConfigOption.class.getDeclaredMethod("getClazz");
+ getClazzMethod.setAccessible(true);
+ Class<?> clazz = (Class<?>) getClazzMethod.invoke(option);
+ getClazzMethod.setAccessible(false);
+ return clazz;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static boolean isList(ConfigOption<?> option) {
+ try {
+ Method getClazzMethod = ConfigOption.class.getDeclaredMethod("isList");
+ getClazzMethod.setAccessible(true);
+ boolean isList = (boolean) getClazzMethod.invoke(option);
+ getClazzMethod.setAccessible(false);
+ return isList;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @VisibleForTesting
+ static String typeToHtml(OptionWithMetaInfo optionWithMetaInfo) {
+ ConfigOption<?> option = optionWithMetaInfo.option;
+ Class<?> clazz = getClazz(option);
+ boolean isList = isList(option);
+ if (clazz.isEnum()) {
+ return enumTypeToHtml(isList);
+ }
+ return atomicTypeToHtml(clazz, isList);
+ }
+
+ private static String atomicTypeToHtml(Class<?> clazz, boolean isList) {
+ String typeName = clazz.getSimpleName();
+
+ final String type;
+ if (isList) {
+ type = String.format("List<%s>", typeName);
+ } else {
+ type = typeName;
+ }
+ return escapeCharacters(type);
+ }
+
+ private static String enumTypeToHtml(boolean isList) {
+ final String type;
+ if (isList) {
+ type = "List<Enum>";
+ } else {
+ type = "Enum";
+ }
+ return String.format("<p>%s</p>", escapeCharacters(type));
+ }
+
+ @VisibleForTesting
+ static String stringifyDefault(OptionWithMetaInfo optionWithMetaInfo) {
+ ConfigOption<?> option = optionWithMetaInfo.option;
+ Documentation.OverrideDefault overrideDocumentedDefault =
+ optionWithMetaInfo.field.getAnnotation(Documentation.OverrideDefault.class);
+ if (overrideDocumentedDefault != null) {
+ return overrideDocumentedDefault.value();
+ } else {
+ Object value = option.defaultValue();
+ return stringifyObject(value);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static String stringifyObject(Object value) {
+ if (value instanceof String) {
+ if (((String) value).isEmpty()) {
+ return "(none)";
+ }
+ return "\"" + value + "\"";
+ } else if (value instanceof Duration) {
+ return TimeUtils.formatWithHighestUnit((Duration) value);
+ } else if (value instanceof List) {
+ return ((List<Object>) value)
+ .stream()
+ .map(ConfigOptionsDocGenerator::stringifyObject)
+ .collect(Collectors.joining(";"));
+ } else if (value instanceof Map) {
+ return ((Map<String, String>) value)
+ .entrySet().stream()
+ .map(e -> String.format("%s:%s", e.getKey(), e.getValue()))
+ .collect(Collectors.joining(","));
+ }
+ return value == null ? "(none)" : value.toString();
+ }
+
+ private static String addWordBreakOpportunities(String value) {
+ return value
+ // allow breaking of semicolon separated lists
+ .replace(";", ";<wbr>");
+ }
+
+ private static void sortOptions(List<OptionWithMetaInfo> configOptions) {
+ configOptions.sort(Comparator.comparing(option -> option.option.key()));
+ }
+
+ /**
+ * Data structure used to assign {@link ConfigOption ConfigOptions} to the {@link ConfigGroup}
+ * with the longest matching prefix.
+ */
+ private static class Tree {
+ private final Node root = new Node();
+
+ Tree(ConfigGroup[] groups, Collection<OptionWithMetaInfo> options) {
+ // generate a tree based on all key prefixes
+ for (ConfigGroup group : groups) {
+ Node currentNode = root;
+ for (String keyComponent : group.keyPrefix().split("\\.")) {
+ currentNode = currentNode.addChild(keyComponent);
+ }
+ currentNode.markAsGroupRoot();
+ }
+
+ // assign options to their corresponding group, i.e. the last group root node
+ // encountered when traversing
+ // the tree based on the option key
+ for (OptionWithMetaInfo option : options) {
+ findGroupRoot(option.option.key()).assignOption(option);
+ }
+ }
+
+ List<OptionWithMetaInfo> findConfigOptions(ConfigGroup configGroup) {
+ Node groupRoot = findGroupRoot(configGroup.keyPrefix());
+ return groupRoot.getConfigOptions();
+ }
+
+ List<OptionWithMetaInfo> getDefaultOptions() {
+ return root.getConfigOptions();
+ }
+
+ private Node findGroupRoot(String key) {
+ Node lastRootNode = root;
+ Node currentNode = root;
+ for (String keyComponent : key.split("\\.")) {
+ final Node childNode = currentNode.getChild(keyComponent);
+ if (childNode == null) {
+ break;
+ } else {
+ currentNode = childNode;
+ if (currentNode.isGroupRoot()) {
+ lastRootNode = currentNode;
+ }
+ }
+ }
+ return lastRootNode;
+ }
+
+ private static class Node {
+ private final List<OptionWithMetaInfo> configOptions = new ArrayList<>(8);
+ private final Map<String, Node> children = new HashMap<>(8);
+ private boolean isGroupRoot = false;
+
+ private Node addChild(String keyComponent) {
+ Node child = children.get(keyComponent);
+ if (child == null) {
+ child = new Node();
+ children.put(keyComponent, child);
+ }
+ return child;
+ }
+
+ private Node getChild(String keyComponent) {
+ return children.get(keyComponent);
+ }
+
+ private void assignOption(OptionWithMetaInfo option) {
+ configOptions.add(option);
+ }
+
+ private boolean isGroupRoot() {
+ return isGroupRoot;
+ }
+
+ private void markAsGroupRoot() {
+ this.isGroupRoot = true;
+ }
+
+ private List<OptionWithMetaInfo> getConfigOptions() {
+ return configOptions;
+ }
+ }
+ }
+
+ static class OptionWithMetaInfo {
+ final ConfigOption<?> option;
+ final Field field;
+
+ public OptionWithMetaInfo(ConfigOption<?> option, Field field) {
+ this.option = option;
+ this.field = field;
+ }
+ }
+
+ private ConfigOptionsDocGenerator() {}
+}
diff --git a/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/OptionsClassLocation.java b/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/OptionsClassLocation.java
new file mode 100644
index 0000000..3384006
--- /dev/null
+++ b/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/OptionsClassLocation.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.docs.configuration;
+
+import org.apache.flink.configuration.ConfigOption;
+
+/** Simple descriptor for the location of a class containing {@link ConfigOption ConfigOptions}. */
+class OptionsClassLocation {
+ private final String module;
+ private final String pckg;
+
+ OptionsClassLocation(String module, String pckg) {
+ this.module = module;
+ this.pckg = pckg;
+ }
+
+ public String getModule() {
+ return module;
+ }
+
+ public String getPackage() {
+ return pckg;
+ }
+
+ @Override
+ public String toString() {
+ return module + "#" + pckg;
+ }
+}
diff --git a/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/util/Utils.java b/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/util/Utils.java
new file mode 100644
index 0000000..fffaf0c
--- /dev/null
+++ b/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/util/Utils.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.docs.util;
+
+/** Contains various shared utility functions. */
+public class Utils {
+
+ /**
+ * Placeholder that is used to prevent certain sections from being escaped. We don't need a
+ * sophisticated value but only something that won't show up in config options.
+ */
+ private static final String TEMPORARY_PLACEHOLDER = "superRandomTemporaryPlaceholder";
+
+ public static String escapeCharacters(String value) {
+ return value.replaceAll("<wbr>", TEMPORARY_PLACEHOLDER)
+ .replaceAll("<", "<")
+ .replaceAll(">", ">")
+ .replaceAll(TEMPORARY_PLACEHOLDER, "<wbr>");
+ }
+}
diff --git a/flink-kubernetes-docs/src/main/resources/log4j.properties b/flink-kubernetes-docs/src/main/resources/log4j.properties
new file mode 100644
index 0000000..e347ecb
--- /dev/null
+++ b/flink-kubernetes-docs/src/main/resources/log4j.properties
@@ -0,0 +1,24 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.logger.org.apache.flink.kubernetes.operator.docs.ConfigOptionsDocGenerator= INFO, logger
+
+log4j.appender.logger=org.apache.log4j.ConsoleAppender
+log4j.appender.logger.target = System.out
+log4j.appender.logger.layout=org.apache.log4j.PatternLayout
+log4j.appender.logger.layout.ConversionPattern=%-5p - %m%n
diff --git a/flink-kubernetes-docs/src/test/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGeneratorTest.java b/flink-kubernetes-docs/src/test/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGeneratorTest.java
new file mode 100644
index 0000000..2f755b0
--- /dev/null
+++ b/flink-kubernetes-docs/src/test/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGeneratorTest.java
@@ -0,0 +1,607 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.docs.configuration;
+
+import org.apache.flink.annotation.docs.ConfigGroup;
+import org.apache.flink.annotation.docs.ConfigGroups;
+import org.apache.flink.annotation.docs.Documentation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.DescribedEnum;
+import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.configuration.description.Formatter;
+import org.apache.flink.configuration.description.HtmlFormatter;
+import org.apache.flink.configuration.description.InlineElement;
+import org.apache.flink.kubernetes.operator.docs.configuration.data.TestCommonOptions;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.flink.configuration.description.TextElement.text;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for the {@link ConfigOptionsDocGenerator}. */
+@SuppressWarnings("unused")
+class ConfigOptionsDocGeneratorTest {
+
+ static class TestConfigGroup {
+ public static ConfigOption<Integer> firstOption =
+ ConfigOptions.key("first.option.a")
+ .intType()
+ .defaultValue(2)
+ .withDescription("This is example description for the first option.");
+
+ public static ConfigOption<String> secondOption =
+ ConfigOptions.key("second.option.a")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("This is long example description for the second option.");
+ }
+
+ private enum TestEnum {
+ VALUE_1,
+ VALUE_2,
+ VALUE_3
+ }
+
+ private enum DescribedTestEnum implements DescribedEnum {
+ A(text("First letter of the alphabet")),
+ B(text("Second letter of the alphabet"));
+
+ private final InlineElement description;
+
+ DescribedTestEnum(InlineElement description) {
+ this.description = description;
+ }
+
+ @Override
+ public InlineElement getDescription() {
+ return description;
+ }
+ }
+
+ static class TypeTestConfigGroup {
+ public static ConfigOption<TestEnum> enumOption =
+ ConfigOptions.key("option.enum")
+ .enumType(TestEnum.class)
+ .defaultValue(TestEnum.VALUE_1)
+ .withDescription("Description");
+
+ public static ConfigOption<List<TestEnum>> enumListOption =
+ ConfigOptions.key("option.enum.list")
+ .enumType(TestEnum.class)
+ .asList()
+ .defaultValues(TestEnum.VALUE_1, TestEnum.VALUE_2)
+ .withDescription("Description");
+
+ public static ConfigOption<DescribedTestEnum> describedEnum =
+ ConfigOptions.key("option.enum.described")
+ .enumType(DescribedTestEnum.class)
+ .noDefaultValue()
+ .withDescription("Description");
+
+ public static ConfigOption<MemorySize> memoryOption =
+ ConfigOptions.key("option.memory")
+ .memoryType()
+ .defaultValue(new MemorySize(1024))
+ .withDescription("Description");
+
+ public static ConfigOption<Map<String, String>> mapOption =
+ ConfigOptions.key("option.map")
+ .mapType()
+ .defaultValue(Collections.singletonMap("key1", "value1"))
+ .withDescription("Description");
+
+ public static ConfigOption<List<Map<String, String>>> mapListOption =
+ ConfigOptions.key("option.map.list")
+ .mapType()
+ .asList()
+ .defaultValues(
+ Collections.singletonMap("key1", "value1"),
+ Collections.singletonMap("key2", "value2"))
+ .withDescription("Description");
+
+ public static ConfigOption<Duration> durationOption =
+ ConfigOptions.key("option.duration")
+ .durationType()
+ .defaultValue(Duration.ofMinutes(1))
+ .withDescription("Description");
+ }
+
+ @Test
+ void testCreatingTypes() {
+ final String expectedTable =
+ "<table class=\"configuration table table-bordered\">\n"
+ + " <thead>\n"
+ + " <tr>\n"
+ + " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n"
+ + " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n"
+ + " <th class=\"text-left\" style=\"width: 10%\">Type</th>\n"
+ + " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n"
+ + " </tr>\n"
+ + " </thead>\n"
+ + " <tbody>\n"
+ + " <tr>\n"
+ + " <td><h5>option.duration</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">1 min</td>\n"
+ + " <td>Duration</td>\n"
+ + " <td>Description</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>option.enum</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">VALUE_1</td>\n"
+ + " <td><p>Enum</p></td>\n"
+ + " <td>Description<br /><br />Possible values:<ul><li>\"VALUE_1\"</li><li>\"VALUE_2\"</li><li>\"VALUE_3\"</li></ul></td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>option.enum.described</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">(none)</td>\n"
+ + " <td><p>Enum</p></td>\n"
+ + " <td>Description<br /><br />Possible values:<ul><li>\"A\": First letter of the alphabet</li><li>\"B\": Second letter of the alphabet</li></ul></td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>option.enum.list</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">VALUE_1;<wbr>VALUE_2</td>\n"
+ + " <td><p>List<Enum></p></td>\n"
+ + " <td>Description<br /><br />Possible values:<ul><li>\"VALUE_1\"</li><li>\"VALUE_2\"</li><li>\"VALUE_3\"</li></ul></td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>option.map</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">key1:value1</td>\n"
+ + " <td>Map</td>\n"
+ + " <td>Description</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>option.map.list</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">key1:value1;<wbr>key2:value2</td>\n"
+ + " <td>List<Map></td>\n"
+ + " <td>Description</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>option.memory</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">1 kb</td>\n"
+ + " <td>MemorySize</td>\n"
+ + " <td>Description</td>\n"
+ + " </tr>\n"
+ + " </tbody>\n"
+ + "</table>\n";
+ final String htmlTable =
+ ConfigOptionsDocGenerator.generateTablesForClass(TypeTestConfigGroup.class)
+ .get(0)
+ .f1;
+ assertThat(htmlTable).isEqualTo(expectedTable);
+ }
+
+ @Test
+ void testCreatingDescription() {
+ final String expectedTable =
+ "<table class=\"configuration table table-bordered\">\n"
+ + " <thead>\n"
+ + " <tr>\n"
+ + " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n"
+ + " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n"
+ + " <th class=\"text-left\" style=\"width: 10%\">Type</th>\n"
+ + " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n"
+ + " </tr>\n"
+ + " </thead>\n"
+ + " <tbody>\n"
+ + " <tr>\n"
+ + " <td><h5>first.option.a</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">2</td>\n"
+ + " <td>Integer</td>\n"
+ + " <td>This is example description for the first option.</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>second.option.a</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">(none)</td>\n"
+ + " <td>String</td>\n"
+ + " <td>This is long example description for the second option.</td>\n"
+ + " </tr>\n"
+ + " </tbody>\n"
+ + "</table>\n";
+ final String htmlTable =
+ ConfigOptionsDocGenerator.generateTablesForClass(TestConfigGroup.class).get(0).f1;
+ assertThat(htmlTable).isEqualTo(expectedTable);
+ }
+
+ @ConfigGroups(
+ groups = {
+ @ConfigGroup(name = "group1", keyPrefix = "a.b"),
+ @ConfigGroup(name = "group2", keyPrefix = "a.b.c.d")
+ })
+ static class TestConfigPrefix {
+ // should end up in the default group
+ public static ConfigOption<Integer> option1 =
+ ConfigOptions.key("a.option").intType().defaultValue(2);
+
+ // should end up in group1, perfect key-prefix match
+ public static ConfigOption<String> option2 =
+ ConfigOptions.key("a.b.option").stringType().noDefaultValue();
+
+ // should end up in group1, full key-prefix match
+ public static ConfigOption<Integer> option3 =
+ ConfigOptions.key("a.b.c.option").intType().defaultValue(2);
+
+ // should end up in group1, full key-prefix match for group 1, partial match for group 2
+ // checks that the generator remembers the last encountered root node
+ public static ConfigOption<Integer> option4 =
+ ConfigOptions.key("a.b.c.e.option").intType().defaultValue(2);
+
+ // should end up in the default group, since no group exists with prefix "a.c"
+ // checks that the generator does not ignore components (like ignoring "c" to find a match
+ // "a.b")
+ public static ConfigOption<String> option5 =
+ ConfigOptions.key("a.c.b.option").stringType().noDefaultValue();
+
+ // should end up in group2, full key-prefix match for group 2
+ // checks that the longest matching group is assigned
+ public static ConfigOption<Integer> option6 =
+ ConfigOptions.key("a.b.c.d.option").intType().defaultValue(2);
+ }
+
+ @Test
+ void testLongestPrefixMatching() {
+ final List<Tuple2<ConfigGroup, String>> tables =
+ ConfigOptionsDocGenerator.generateTablesForClass(TestConfigPrefix.class);
+ assertThat(tables).hasSize(3);
+ final Map<String, String> tablesConverted = new HashMap<>(tables.size());
+ for (final Tuple2<ConfigGroup, String> table : tables) {
+ tablesConverted.put(table.f0 != null ? table.f0.name() : "default", table.f1);
+ }
+ assertThat(tablesConverted.get("group1"))
+ .contains("a.b.option", "a.b.c.option", "a.b.c.e.option")
+ .doesNotContain("a.b.c.d.option");
+ assertThat(tablesConverted.get("group2")).contains("a.b.c.d.option");
+ assertThat(tablesConverted.get("default")).contains("a.option", "a.c.b.option");
+ }
+
+ @ConfigGroups(
+ groups = {
+ @ConfigGroup(name = "firstGroup", keyPrefix = "first"),
+ @ConfigGroup(name = "secondGroup", keyPrefix = "second")
+ })
+ static class TestConfigMultipleSubGroup {
+ public static ConfigOption<Integer> firstOption =
+ ConfigOptions.key("first.option.a")
+ .intType()
+ .defaultValue(2)
+ .withDescription("This is example description for the first option.");
+
+ public static ConfigOption<String> secondOption =
+ ConfigOptions.key("second.option.a")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("This is long example description for the second option.");
+
+ public static ConfigOption<Integer> thirdOption =
+ ConfigOptions.key("third.option.a")
+ .intType()
+ .defaultValue(2)
+ .withDescription("This is example description for the third option.");
+
+ public static ConfigOption<String> fourthOption =
+ ConfigOptions.key("fourth.option.a")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("This is long example description for the fourth option.");
+ }
+
+ @Test
+ void testCreatingMultipleGroups() {
+ final List<Tuple2<ConfigGroup, String>> tables =
+ ConfigOptionsDocGenerator.generateTablesForClass(TestConfigMultipleSubGroup.class);
+ assertThat(tables).hasSize(3);
+ final HashMap<String, String> tablesConverted = new HashMap<>();
+ for (Tuple2<ConfigGroup, String> table : tables) {
+ tablesConverted.put(table.f0 != null ? table.f0.name() : "default", table.f1);
+ }
+ assertThat(tablesConverted)
+ .containsEntry(
+ "firstGroup",
+ "<table class=\"configuration table table-bordered\">\n"
+ + " <thead>\n"
+ + " <tr>\n"
+ + " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n"
+ + " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n"
+ + " <th class=\"text-left\" style=\"width: 10%\">Type</th>\n"
+ + " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n"
+ + " </tr>\n"
+ + " </thead>\n"
+ + " <tbody>\n"
+ + " <tr>\n"
+ + " <td><h5>first.option.a</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">2</td>\n"
+ + " <td>Integer</td>\n"
+ + " <td>This is example description for the first option.</td>\n"
+ + " </tr>\n"
+ + " </tbody>\n"
+ + "</table>\n");
+ assertThat(tablesConverted)
+ .containsEntry(
+ "secondGroup",
+ "<table class=\"configuration table table-bordered\">\n"
+ + " <thead>\n"
+ + " <tr>\n"
+ + " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n"
+ + " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n"
+ + " <th class=\"text-left\" style=\"width: 10%\">Type</th>\n"
+ + " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n"
+ + " </tr>\n"
+ + " </thead>\n"
+ + " <tbody>\n"
+ + " <tr>\n"
+ + " <td><h5>second.option.a</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">(none)</td>\n"
+ + " <td>String</td>\n"
+ + " <td>This is long example description for the second option.</td>\n"
+ + " </tr>\n"
+ + " </tbody>\n"
+ + "</table>\n");
+ assertThat(tablesConverted)
+ .containsEntry(
+ "default",
+ "<table class=\"configuration table table-bordered\">\n"
+ + " <thead>\n"
+ + " <tr>\n"
+ + " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n"
+ + " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n"
+ + " <th class=\"text-left\" style=\"width: 10%\">Type</th>\n"
+ + " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n"
+ + " </tr>\n"
+ + " </thead>\n"
+ + " <tbody>\n"
+ + " <tr>\n"
+ + " <td><h5>fourth.option.a</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">(none)</td>\n"
+ + " <td>String</td>\n"
+ + " <td>This is long example description for the fourth option.</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>third.option.a</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">2</td>\n"
+ + " <td>Integer</td>\n"
+ + " <td>This is example description for the third option.</td>\n"
+ + " </tr>\n"
+ + " </tbody>\n"
+ + "</table>\n");
+ }
+
+ static class TestConfigGroupWithOverriddenDefault {
+ @Documentation.OverrideDefault("default_1")
+ public static ConfigOption<Integer> firstOption =
+ ConfigOptions.key("first.option.a")
+ .intType()
+ .defaultValue(2)
+ .withDescription("This is example description for the first option.");
+
+ @Documentation.OverrideDefault("default_2")
+ public static ConfigOption<String> secondOption =
+ ConfigOptions.key("second.option.a")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("This is long example description for the second option.");
+ }
+
+ @Test
+ void testOverrideDefault() {
+ final String expectedTable =
+ "<table class=\"configuration table table-bordered\">\n"
+ + " <thead>\n"
+ + " <tr>\n"
+ + " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n"
+ + " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n"
+ + " <th class=\"text-left\" style=\"width: 10%\">Type</th>\n"
+ + " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n"
+ + " </tr>\n"
+ + " </thead>\n"
+ + " <tbody>\n"
+ + " <tr>\n"
+ + " <td><h5>first.option.a</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">default_1</td>\n"
+ + " <td>Integer</td>\n"
+ + " <td>This is example description for the first option.</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>second.option.a</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">default_2</td>\n"
+ + " <td>String</td>\n"
+ + " <td>This is long example description for the second option.</td>\n"
+ + " </tr>\n"
+ + " </tbody>\n"
+ + "</table>\n";
+ final String htmlTable =
+ ConfigOptionsDocGenerator.generateTablesForClass(
+ TestConfigGroupWithOverriddenDefault.class)
+ .get(0)
+ .f1;
+ assertThat(htmlTable).isEqualTo(expectedTable);
+ }
+
+ @Test
+ void testSections(@TempDir File tmpDir) throws IOException, ClassNotFoundException {
+ final String projectRootDir = getProjectRootDir();
+ final String outputDirectory = tmpDir.getAbsolutePath();
+ final OptionsClassLocation[] locations =
+ new OptionsClassLocation[] {
+ new OptionsClassLocation(
+ "flink-kubernetes-docs", TestCommonOptions.class.getPackage().getName())
+ };
+ ConfigOptionsDocGenerator.generateCommonSection(
+ projectRootDir, outputDirectory, locations, "src/test/java");
+ Formatter formatter = new HtmlFormatter();
+ String expected1 =
+ "<table class=\"configuration table table-bordered\">\n"
+ + " <thead>\n"
+ + " <tr>\n"
+ + " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n"
+ + " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n"
+ + " <th class=\"text-left\" style=\"width: 10%\">Type</th>\n"
+ + " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n"
+ + " </tr>\n"
+ + " </thead>\n"
+ + " <tbody>\n"
+ + " <tr>\n"
+ + " <td><h5>"
+ + TestCommonOptions.COMMON_POSITIONED_OPTION.key()
+ + "</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">"
+ + TestCommonOptions.COMMON_POSITIONED_OPTION.defaultValue()
+ + "</td>\n"
+ + " <td>Integer</td>\n"
+ + " <td>"
+ + formatter.format(TestCommonOptions.COMMON_POSITIONED_OPTION.description())
+ + "</td>\n"
+ + " </tr>\n"
+ + " <tr>\n"
+ + " <td><h5>"
+ + TestCommonOptions.COMMON_OPTION.key()
+ + "</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">"
+ + TestCommonOptions.COMMON_OPTION.defaultValue()
+ + "</td>\n"
+ + " <td>Integer</td>\n"
+ + " <td>"
+ + formatter.format(TestCommonOptions.COMMON_OPTION.description())
+ + "</td>\n"
+ + " </tr>\n"
+ + " </tbody>\n"
+ + "</table>\n";
+ String expected2 =
+ "<table class=\"configuration table table-bordered\">\n"
+ + " <thead>\n"
+ + " <tr>\n"
+ + " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n"
+ + " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n"
+ + " <th class=\"text-left\" style=\"width: 10%\">Type</th>\n"
+ + " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n"
+ + " </tr>\n"
+ + " </thead>\n"
+ + " <tbody>\n"
+ + " <tr>\n"
+ + " <td><h5>"
+ + TestCommonOptions.COMMON_OPTION.key()
+ + "</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">"
+ + TestCommonOptions.COMMON_OPTION.defaultValue()
+ + "</td>\n"
+ + " <td>Integer</td>\n"
+ + " <td>"
+ + formatter.format(TestCommonOptions.COMMON_OPTION.description())
+ + "</td>\n"
+ + " </tr>\n"
+ + " </tbody>\n"
+ + "</table>\n";
+ final String fileName1 =
+ ConfigOptionsDocGenerator.getSectionFileName(TestCommonOptions.SECTION_1);
+ final String fileName2 =
+ ConfigOptionsDocGenerator.getSectionFileName(TestCommonOptions.SECTION_2);
+ assertThat(new File(outputDirectory, fileName1))
+ .content(StandardCharsets.UTF_8)
+ .isEqualTo(expected1);
+ assertThat(new File(outputDirectory, fileName2))
+ .content(StandardCharsets.UTF_8)
+ .isEqualTo(expected2);
+ }
+
+ static class TestConfigGroupWithExclusion {
+ public static ConfigOption<Integer> firstOption =
+ ConfigOptions.key("first.option.a")
+ .intType()
+ .defaultValue(2)
+ .withDescription("This is example description for the first option.");
+
+ @Documentation.ExcludeFromDocumentation
+ public static ConfigOption<String> excludedOption =
+ ConfigOptions.key("excluded.option.a")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("This should not be documented.");
+ }
+
+ /**
+ * Tests that {@link ConfigOption} annotated with {@link Documentation.ExcludeFromDocumentation}
+ * are not documented.
+ */
+ @Test
+ void testConfigOptionExclusion() {
+ final String expectedTable =
+ "<table class=\"configuration table table-bordered\">\n"
+ + " <thead>\n"
+ + " <tr>\n"
+ + " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n"
+ + " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n"
+ + " <th class=\"text-left\" style=\"width: 10%\">Type</th>\n"
+ + " <th class=\"text-left\" style=\"width: 55%\">Description</th>\n"
+ + " </tr>\n"
+ + " </thead>\n"
+ + " <tbody>\n"
+ + " <tr>\n"
+ + " <td><h5>first.option.a</h5></td>\n"
+ + " <td style=\"word-wrap: break-word;\">2</td>\n"
+ + " <td>Integer</td>\n"
+ + " <td>This is example description for the first option.</td>\n"
+ + " </tr>\n"
+ + " </tbody>\n"
+ + "</table>\n";
+ final String htmlTable =
+ ConfigOptionsDocGenerator.generateTablesForClass(TestConfigGroupWithExclusion.class)
+ .get(0)
+ .f1;
+ assertThat(htmlTable).isEqualTo(expectedTable);
+ }
+
+ @ConfigGroups(groups = {@ConfigGroup(name = "firstGroup", keyPrefix = "first")})
+ static class EmptyConfigOptions {}
+
+ @Test
+ void testClassWithoutOptionsIsIgnored() {
+ assertThat(ConfigOptionsDocGenerator.generateTablesForClass(EmptyConfigOptions.class))
+ .isEmpty();
+ }
+
+ @Test
+ void testSnakeCaseConversion() {
+ assertThat(ConfigOptionsDocGenerator.toSnakeCase("RocksOptions"))
+ .isEqualTo("rocks_options");
+ assertThat(ConfigOptionsDocGenerator.toSnakeCase("RocksDBOptions"))
+ .isEqualTo("rocksdb_options");
+ assertThat(ConfigOptionsDocGenerator.toSnakeCase("DBOptions")).isEqualTo("db_options");
+ }
+
+ static String getProjectRootDir() {
+ final String dirFromProperty = System.getProperty("rootDir");
+ if (dirFromProperty != null) {
+ return dirFromProperty;
+ }
+
+ // to make this work in the IDE use a default fallback
+ final String currentDir = System.getProperty("user.dir");
+ return new File(currentDir).getParent();
+ }
+}
diff --git a/flink-kubernetes-docs/src/test/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocsCompletenessITCase.java b/flink-kubernetes-docs/src/test/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocsCompletenessITCase.java
new file mode 100644
index 0000000..f1e267b
--- /dev/null
+++ b/flink-kubernetes-docs/src/test/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocsCompletenessITCase.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.docs.configuration;
+
+import org.apache.flink.annotation.docs.Documentation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.ConfigOption;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.flink.kubernetes.operator.docs.configuration.ConfigOptionsDocGenerator.DEFAULT_PATH_PREFIX;
+import static org.apache.flink.kubernetes.operator.docs.configuration.ConfigOptionsDocGenerator.LOCATIONS;
+import static org.apache.flink.kubernetes.operator.docs.configuration.ConfigOptionsDocGenerator.extractConfigOptions;
+import static org.apache.flink.kubernetes.operator.docs.configuration.ConfigOptionsDocGenerator.getDescription;
+import static org.apache.flink.kubernetes.operator.docs.configuration.ConfigOptionsDocGenerator.processConfigOptions;
+import static org.apache.flink.kubernetes.operator.docs.configuration.ConfigOptionsDocGenerator.stringifyDefault;
+import static org.apache.flink.kubernetes.operator.docs.configuration.ConfigOptionsDocGenerator.typeToHtml;
+import static org.assertj.core.api.Fail.fail;
+
+/**
+ * This test verifies that all {@link ConfigOption ConfigOptions} in the configured {@link
+ * ConfigOptionsDocGenerator#LOCATIONS locations} are documented and well-defined (i.e. no 2 options
+ * exist for the same key with different descriptions/default values), and that the documentation
+ * does not refer to non-existent options.
+ */
+public class ConfigOptionsDocsCompletenessITCase {
+
+ @Test
+ public void testCompleteness() throws IOException, ClassNotFoundException {
+ compareDocumentedAndExistingOptions(
+ parseDocumentedOptions(),
+ checkWellDefinedAndDeduplicate(findExistingOptions(ignored -> true)));
+ }
+
+ private static Map<String, List<ExistingOption>> checkWellDefinedAndDeduplicate(
+ Map<String, List<ExistingOption>> allOptions) {
+ return allOptions.entrySet().stream()
+ .map(
+ (entry) -> {
+ final List<ExistingOption> existingOptions = entry.getValue();
+ final List<ExistingOption> consolidated;
+ if (existingOptions.stream()
+ .allMatch(option -> option.isSuffixOption)) {
+ consolidated = existingOptions;
+ } else {
+ consolidated =
+ Collections.singletonList(
+ existingOptions.stream()
+ .reduce(
+ (option1, option2) -> {
+ if (option1.equals(option2)) {
+ // we allow multiple
+ // instances of
+ // ConfigOptions with the
+ // same key
+ // if they are identical
+ return option1;
+ } else {
+ // found a ConfigOption pair
+ // with
+ // the same key that aren't
+ // equal
+ // we fail here outright as
+ // this is
+ // not a
+ // documentation-completeness
+ // problem
+ String errorMessage;
+ if (!option1.defaultValue
+ .equals(
+ option2.defaultValue)) {
+ errorMessage =
+ String.format(
+ "Ambiguous option %s due to distinct default values (%s (in %s) vs %s (in %s)).",
+ option1.key,
+ option1.defaultValue,
+ option1
+ .containingClass
+ .getSimpleName(),
+ option2.defaultValue,
+ option2
+ .containingClass
+ .getSimpleName());
+ } else {
+ errorMessage =
+ String.format(
+ "Ambiguous option %s due to distinct descriptions (%s vs %s).",
+ option1.key,
+ option1
+ .containingClass
+ .getSimpleName(),
+ option2
+ .containingClass
+ .getSimpleName());
+ }
+ throw new AssertionError(
+ errorMessage);
+ }
+ })
+ .get());
+ }
+ return new Tuple2<>(entry.getKey(), consolidated);
+ })
+ .collect(Collectors.toMap((t) -> t.f0, (t) -> t.f1));
+ }
+
+ private static void compareDocumentedAndExistingOptions(
+ Map<String, List<DocumentedOption>> documentedOptions,
+ Map<String, List<ExistingOption>> existingOptions) {
+ final Collection<String> problems = new ArrayList<>(0);
+ // first check that all existing options are properly documented
+ existingOptions.forEach(
+ (key, supposedStates) -> {
+ List<DocumentedOption> documentedState = documentedOptions.get(key);
+ for (ExistingOption supposedState : supposedStates) {
+ if (documentedState == null || documentedState.isEmpty()) {
+ // option is not documented at all
+ problems.add(
+ "Option "
+ + supposedState.key
+ + " in "
+ + supposedState.containingClass
+ + " is not documented.");
+ } else {
+ final Iterator<DocumentedOption> candidates =
+ documentedState.iterator();
+ boolean matchFound = false;
+ while (candidates.hasNext()) {
+ DocumentedOption candidate = candidates.next();
+ if (supposedState.defaultValue.equals(candidate.defaultValue)
+ && supposedState.description.equals(
+ candidate.description)) {
+ matchFound = true;
+ candidates.remove();
+ }
+ }
+ if (documentedState.isEmpty()) {
+ documentedOptions.remove(key);
+ }
+ if (!matchFound) {
+ problems.add(
+ String.format(
+ "Documentation of %s in %s is outdated. Expected: default=(%s) description=(%s).",
+ supposedState.key,
+ supposedState.containingClass.getSimpleName(),
+ supposedState.defaultValue,
+ supposedState.description));
+ }
+ }
+ }
+ });
+
+ // documentation contains an option that no longer exists
+ documentedOptions.values().stream()
+ .flatMap(Collection::stream)
+ .forEach(
+ documentedOption ->
+ problems.add(
+ "Documented option "
+ + documentedOption.key
+ + " does not exist."));
+ if (!problems.isEmpty()) {
+ StringBuilder stringBuilder =
+ new StringBuilder(
+ "Documentation is outdated, please regenerate it according to the"
+ + " instructions in flink-kubernetes-docs/README.md.");
+ stringBuilder.append(System.lineSeparator());
+ stringBuilder.append("\tProblems:");
+ for (String problem : problems) {
+ stringBuilder.append(System.lineSeparator());
+ stringBuilder.append("\t\t");
+ stringBuilder.append(problem);
+ }
+ fail(stringBuilder.toString());
+ }
+ }
+
+ private static Map<String, List<DocumentedOption>> parseDocumentedOptions() throws IOException {
+ final String rootDir = ConfigOptionsDocGeneratorTest.getProjectRootDir();
+ Path includeFolder =
+ Paths.get(rootDir, "docs", "layouts", "shortcodes", "generated").toAbsolutePath();
+ return Files.list(includeFolder)
+ .filter(
+ (path) -> {
+ final String filename = path.getFileName().toString();
+ return filename.endsWith("configuration.html");
+ })
+ .flatMap(
+ file -> {
+ try {
+ return parseDocumentedOptionsFromFile(file).stream();
+ } catch (IOException ignored) {
+ return Stream.empty();
+ }
+ })
+ .collect(Collectors.groupingBy(option -> option.key, Collectors.toList()));
+ }
+
+ private static Collection<DocumentedOption> parseDocumentedOptionsFromFile(Path file)
+ throws IOException {
+ Document document = Jsoup.parse(file.toFile(), StandardCharsets.UTF_8.name());
+ document.outputSettings().syntax(Document.OutputSettings.Syntax.xml);
+ document.outputSettings().prettyPrint(false);
+ return document.getElementsByTag("table").stream()
+ .map(element -> element.getElementsByTag("tbody").get(0))
+ .flatMap(element -> element.getElementsByTag("tr").stream())
+ .map(
+ tableRow -> {
+ // Use split to exclude document key tag.
+ String key = tableRow.child(0).text().split(" ")[0];
+ String defaultValue = tableRow.child(1).text();
+ String typeValue = tableRow.child(2).html();
+ String description =
+ tableRow.child(3).childNodes().stream()
+ .map(Object::toString)
+ .collect(Collectors.joining());
+ return new DocumentedOption(key, defaultValue, typeValue, description);
+ })
+ .collect(Collectors.toList());
+ }
+
+ private static Map<String, List<ExistingOption>> findExistingOptions(
+ Predicate<ConfigOptionsDocGenerator.OptionWithMetaInfo> predicate)
+ throws IOException, ClassNotFoundException {
+ final String rootDir = ConfigOptionsDocGeneratorTest.getProjectRootDir();
+ final Collection<ExistingOption> existingOptions = new ArrayList<>();
+ for (OptionsClassLocation location : LOCATIONS) {
+ processConfigOptions(
+ rootDir,
+ location.getModule(),
+ location.getPackage(),
+ DEFAULT_PATH_PREFIX,
+ optionsClass ->
+ extractConfigOptions(optionsClass).stream()
+ .filter(predicate)
+ .map(
+ optionWithMetaInfo ->
+ toExistingOption(
+ optionWithMetaInfo, optionsClass))
+ .forEach(existingOptions::add));
+ }
+ return existingOptions.stream()
+ .collect(Collectors.groupingBy(option -> option.key, Collectors.toList()));
+ }
+
+ private static ExistingOption toExistingOption(
+ ConfigOptionsDocGenerator.OptionWithMetaInfo optionWithMetaInfo,
+ Class<?> optionsClass) {
+ String key = optionWithMetaInfo.option.key();
+ String defaultValue = stringifyDefault(optionWithMetaInfo);
+ String typeValue = typeToHtml(optionWithMetaInfo);
+ String description = getDescription(optionWithMetaInfo);
+ boolean isSuffixOption = isSuffixOption(optionWithMetaInfo.field);
+ return new ExistingOption(
+ key, defaultValue, typeValue, description, optionsClass, isSuffixOption);
+ }
+
+ private static boolean isSuffixOption(Field field) {
+ final Class<?> containingOptionsClass = field.getDeclaringClass();
+
+ return field.getAnnotation(Documentation.SuffixOption.class) != null
+ || containingOptionsClass.getAnnotation(Documentation.SuffixOption.class) != null;
+ }
+
+ private static final class ExistingOption extends Option {
+
+ private final Class<?> containingClass;
+ private final boolean isSuffixOption;
+
+ private ExistingOption(
+ String key,
+ String defaultValue,
+ String typeValue,
+ String description,
+ Class<?> containingClass,
+ boolean isSuffixOption) {
+ super(key, defaultValue, typeValue, description);
+ this.containingClass = containingClass;
+ this.isSuffixOption = isSuffixOption;
+ }
+ }
+
+ private static final class DocumentedOption extends Option {
+
+ private DocumentedOption(
+ String key, String defaultValue, String typeValue, String description) {
+ super(key, defaultValue, typeValue, description);
+ }
+ }
+
+ private abstract static class Option {
+ protected final String key;
+ protected final String defaultValue;
+ protected final String typeValue;
+ protected final String description;
+
+ private Option(String key, String defaultValue, String typeValue, String description) {
+ this.key = key;
+ this.defaultValue = defaultValue;
+ this.typeValue = typeValue;
+ this.description = description;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Option option = (Option) o;
+ return Objects.equals(key, option.key)
+ && Objects.equals(defaultValue, option.defaultValue)
+ && Objects.equals(typeValue, option.typeValue)
+ && Objects.equals(description, option.description);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(key, defaultValue, typeValue, description);
+ }
+
+ @Override
+ public String toString() {
+ return "Option{"
+ + "key='"
+ + key
+ + '\''
+ + ", defaultValue='"
+ + defaultValue
+ + '\''
+ + ", typeValue='"
+ + typeValue
+ + '\''
+ + ", description='"
+ + description
+ + '\''
+ + '}';
+ }
+ }
+}
diff --git a/flink-kubernetes-docs/src/test/java/org/apache/flink/kubernetes/operator/docs/configuration/data/TestCommonOptions.java b/flink-kubernetes-docs/src/test/java/org/apache/flink/kubernetes/operator/docs/configuration/data/TestCommonOptions.java
new file mode 100644
index 0000000..0c9668d
--- /dev/null
+++ b/flink-kubernetes-docs/src/test/java/org/apache/flink/kubernetes/operator/docs/configuration/data/TestCommonOptions.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.docs.configuration.data;
+
+import org.apache.flink.annotation.docs.Documentation;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+
+/** Collection of test {@link ConfigOptions ConfigOptions}. */
+@SuppressWarnings("unused") // this class is only accessed reflectively
+public class TestCommonOptions {
+
+ public static final String SECTION_1 = "test_A";
+ public static final String SECTION_2 = "other";
+
+ @Documentation.Section({SECTION_1, SECTION_2})
+ public static final ConfigOption<Integer> COMMON_OPTION =
+ ConfigOptions.key("first.option.a")
+ .intType()
+ .defaultValue(2)
+ .withDescription("This is the description for the common option.");
+
+ public static final ConfigOption<String> GENERIC_OPTION =
+ ConfigOptions.key("second.option.a")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("This is the description for the generic option.");
+
+ @Documentation.Section(value = SECTION_1, position = 2)
+ public static final ConfigOption<Integer> COMMON_POSITIONED_OPTION =
+ ConfigOptions.key("third.option.a")
+ .intType()
+ .defaultValue(3)
+ .withDescription("This is the description for the positioned common option.");
+}
diff --git a/flink-kubernetes-webhook/pom.xml b/flink-kubernetes-webhook/pom.xml
index 189ee2a..fe27466 100644
--- a/flink-kubernetes-webhook/pom.xml
+++ b/flink-kubernetes-webhook/pom.xml
@@ -31,10 +31,6 @@ under the License.
<name>Flink Kubernetes Webhook</name>
<packaging>jar</packaging>
- <properties>
- <junit.jupiter.version>5.8.2</junit.jupiter.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
diff --git a/pom.xml b/pom.xml
index 380e0ee..bd1019d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,6 +54,7 @@ under the License.
<module>flink-kubernetes-shaded</module>
<module>flink-kubernetes-operator</module>
<module>flink-kubernetes-webhook</module>
+ <module>flink-kubernetes-docs</module>
</modules>
<properties>
@@ -75,6 +76,7 @@ under the License.
<slf4j.version>1.7.36</slf4j.version>
<log4j.version>2.17.1</log4j.version>
+ <junit.jupiter.version>5.8.2</junit.jupiter.version>
<spotless.version>2.4.2</spotless.version>
<it.skip>true</it.skip>
@@ -272,6 +274,7 @@ under the License.
<exclude>docs/themes/book/**</exclude>
<exclude>docs/assets/github.css</exclude>
<exclude>docs/static/js/anchor.min.js</exclude>
+ <exclude>docs/layouts/shortcodes/generated/**</exclude>
<!-- PyCharm -->
<exclude>**/.idea/**</exclude>
<!-- Generated CRD -->
@@ -285,6 +288,29 @@ under the License.
</excludes>
</configuration>
</plugin>
+
+ <!-- generate configuration docs -->
+ <plugin>
+ <groupId>org.commonjava.maven.plugins</groupId>
+ <artifactId>directory-maven-plugin</artifactId>
+ <version>0.1</version>
+ <executions>
+ <execution>
+ <id>directories</id>
+ <goals>
+ <goal>directory-of</goal>
+ </goals>
+ <phase>initialize</phase>
+ <configuration>
+ <property>rootDir</property>
+ <project>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-kubernetes-operator-parent</artifactId>
+ </project>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>