You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/04/12 13:49:02 UTC
[1/2] flink git commit: [FLINK-8684][mesos] Make MESOS_RM_TASKS_SLOTS
an alias
Repository: flink
Updated Branches:
refs/heads/master 0a5a64ab7 -> 612e2d834
[FLINK-8684][mesos] Make MESOS_RM_TASKS_SLOTS an alias
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a926bc69
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a926bc69
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a926bc69
Branch: refs/heads/master
Commit: a926bc69416059acca91b13508bf1b159d921c73
Parents: 0a5a64a
Author: zentol <ch...@apache.org>
Authored: Fri Feb 16 22:42:51 2018 +0100
Committer: zentol <ch...@apache.org>
Committed: Thu Apr 12 11:57:53 2018 +0200
----------------------------------------------------------------------
docs/_includes/generated/mesos_task_manager_configuration.html | 2 +-
.../runtime/clusterframework/MesosTaskManagerParameters.java | 5 ++---
2 files changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a926bc69/docs/_includes/generated/mesos_task_manager_configuration.html
----------------------------------------------------------------------
diff --git a/docs/_includes/generated/mesos_task_manager_configuration.html b/docs/_includes/generated/mesos_task_manager_configuration.html
index 182418d..a6ee78b 100644
--- a/docs/_includes/generated/mesos_task_manager_configuration.html
+++ b/docs/_includes/generated/mesos_task_manager_configuration.html
@@ -65,7 +65,7 @@
<tr>
<td><h5>taskmanager.numberOfTaskSlots</h5></td>
<td style="word-wrap: break-word;">1</td>
- <td></td>
+ <td>The number of parallel operator or user function instances that a single TaskManager can run. If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores).</td>
</tr>
</tbody>
</table>
http://git-wip-us.apache.org/repos/asf/flink/blob/a926bc69/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
index 68061d8..6dbfc93 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
@@ -18,10 +18,10 @@
package org.apache.flink.mesos.runtime.clusterframework;
-import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.TaskManagerOptions;
import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
import org.apache.flink.util.Preconditions;
@@ -51,8 +51,7 @@ public class MesosTaskManagerParameters {
public static final Pattern TASK_ID_PATTERN = Pattern.compile("_TASK_", Pattern.LITERAL);
public static final ConfigOption<Integer> MESOS_RM_TASKS_SLOTS =
- key(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS)
- .defaultValue(1);
+ TaskManagerOptions.NUM_TASK_SLOTS;
public static final ConfigOption<Integer> MESOS_RM_TASKS_MEMORY_MB =
key("mesos.resourcemanager.tasks.mem")
[2/2] flink git commit: [hotfix][docs] Refactor
ConfigOptionsDocGenerator
Posted by ch...@apache.org.
[hotfix][docs] Refactor ConfigOptionsDocGenerator
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/612e2d83
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/612e2d83
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/612e2d83
Branch: refs/heads/master
Commit: 612e2d83402ba2c3eb68d7a26f1b5f22b283583e
Parents: a926bc6
Author: zentol <ch...@apache.org>
Authored: Fri Feb 16 23:09:54 2018 +0100
Committer: zentol <ch...@apache.org>
Committed: Thu Apr 12 11:58:18 2018 +0200
----------------------------------------------------------------------
flink-docs/README.md | 2 +-
flink-docs/pom.xml | 62 ++++++------
.../ConfigOptionsDocGenerator.java | 99 ++++++++++++--------
.../configuration/OptionsClassLocation.java | 42 +++++++++
4 files changed, 134 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/612e2d83/flink-docs/README.md
----------------------------------------------------------------------
diff --git a/flink-docs/README.md b/flink-docs/README.md
index bad5444..61624fa 100644
--- a/flink-docs/README.md
+++ b/flink-docs/README.md
@@ -39,7 +39,7 @@ The documentation must be regenerated whenever
The `ConfigOptionsDocGenerator` can be use to generate a reference of `ConfigOptions`. By default, a separate file is generated for each `*Options` class found in `org.apache.flink.configuration` and `org.apache.flink.yarn.configuration`. The `@ConfigGroups` annotation can be used to generate multiple files from a single class.
-To integrate an `*Options` class from another package, add another module-package argument pair to the `maven-antrun-plugin` configuration in the `generate-config-docs` profile.
+To integrate an `*Options` class from another package, add another module-package argument pair to `ConfigOptionsDocGenerator#LOCATIONS`.
The files can be generated by running `mvn package -Dgenerate-config-docs -pl flink-docs -am -nsu`, and can be integrated into the documentation using `{% include generated/<file-name>.html %}`.
http://git-wip-us.apache.org/repos/asf/flink/blob/612e2d83/flink-docs/pom.xml
----------------------------------------------------------------------
diff --git a/flink-docs/pom.xml b/flink-docs/pom.xml
index b06e587..f98de57 100644
--- a/flink-docs/pom.xml
+++ b/flink-docs/pom.xml
@@ -118,6 +118,15 @@ under the License.
</executions>
</plugin>
</plugins>
+
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
</build>
<profiles>
@@ -132,24 +141,24 @@ under the License.
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
<executions>
<execution>
+ <id>generate-rest-docs</id>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
+ <configuration>
+ <target>
+ <mkdir dir="${rootDir}/${generated.docs.dir}"/>
+ <java classname="org.apache.flink.docs.rest.RestAPIDocGenerator" fork="true" failonerror="true">
+ <classpath refid="maven.compile.classpath"/>
+ <arg value="${rootDir}/${generated.docs.dir}/"/>
+ </java>
+ </target>
+ </configuration>
</execution>
</executions>
- <configuration>
- <target>
- <mkdir dir="${rootDir}/${generated.docs.dir}"/>
- <java classname="org.apache.flink.docs.rest.RestAPIDocGenerator" fork="true">
- <classpath refid="maven.compile.classpath"/>
- <arg value="${rootDir}/${generated.docs.dir}/"/>
- </java>
- </target>
- </configuration>
</plugin>
</plugins>
</build>
@@ -165,36 +174,25 @@ under the License.
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
<executions>
<execution>
+ <id>generate-config-docs</id>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
+ <configuration>
+ <target>
+ <mkdir dir="${rootDir}/${generated.docs.dir}"/>
+ <java classname="org.apache.flink.docs.configuration.ConfigOptionsDocGenerator" fork="true" failonerror="true">
+ <classpath refid="maven.compile.classpath" />
+ <arg value="${rootDir}/${generated.docs.dir}/" />
+ <arg value="${rootDir}" />
+ </java>
+ </target>
+ </configuration>
</execution>
</executions>
- <configuration>
- <target>
- <mkdir dir="${rootDir}/${generated.docs.dir}"/>
- <java classname="org.apache.flink.docs.configuration.ConfigOptionsDocGenerator" fork="true">
- <classpath refid="maven.compile.classpath" />
- <arg value="${rootDir}/${generated.docs.dir}/" />
- <arg value="${rootDir}" />
- <!--packages with configuration classes-->
- <arg value="flink-core" />
- <arg value="org.apache.flink.configuration" />
- <arg value="flink-runtime" />
- <arg value="org.apache.flink.runtime.io.network.netty" />
- <arg value="flink-yarn" />
- <arg value="org.apache.flink.yarn.configuration" />
- <arg value="flink-mesos" />
- <arg value="org.apache.flink.mesos.configuration" />
- <arg value="flink-mesos" />
- <arg value="org.apache.flink.mesos.runtime.clusterframework" />
- </java>
- </target>
- </configuration>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/flink/blob/612e2d83/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
----------------------------------------------------------------------
diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
index 3bec09c..6256a67 100644
--- a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
+++ b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.CoreOptions;
import org.apache.flink.configuration.WebOptions;
+import org.apache.flink.util.function.ThrowingConsumer;
import java.io.IOException;
import java.lang.reflect.Field;
@@ -35,6 +36,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
@@ -47,6 +49,18 @@ import java.util.regex.Pattern;
*/
public class ConfigOptionsDocGenerator {
+ private static final OptionsClassLocation[] LOCATIONS = new OptionsClassLocation[]{
+ new OptionsClassLocation("flink-core", "org.apache.flink.configuration"),
+ new OptionsClassLocation("flink-runtime", "org.apache.flink.runtime.io.network.netty"),
+ new OptionsClassLocation("flink-yarn", "org.apache.flink.yarn.configuration"),
+ new OptionsClassLocation("flink-mesos", "org.apache.flink.mesos.configuration"),
+ new OptionsClassLocation("flink-mesos", "org.apache.flink.mesos.runtime.clusterframework"),
+ };
+
+ private static final String CLASS_NAME_GROUP = "className";
+ private static final String CLASS_PREFIX_GROUP = "classPrefix";
+ private static final Pattern CLASS_NAME_PATTERN = Pattern.compile("(?<" + CLASS_NAME_GROUP + ">(?<" + CLASS_PREFIX_GROUP + ">[a-zA-Z]*)(?:Options|Config|Parameters))(?:\\.java)?");
+
/**
* This method generates html tables from set of classes containing {@link ConfigOption ConfigOptions}.
*
@@ -57,39 +71,47 @@ public class ConfigOptionsDocGenerator {
* @param args
* [0] output directory for the generated files
* [1] project root directory
- * [x] module containing an *Options class
- * [x+1] package to the * Options.classes
*/
public static void main(String[] args) throws IOException, ClassNotFoundException {
String outputDirectory = args[0];
String rootDir = args[1];
- for (int x = 2; x + 1 < args.length; x += 2) {
- createTable(rootDir, args[x], args[x + 1], outputDirectory);
+
+ for (OptionsClassLocation location : LOCATIONS) {
+ createTable(rootDir, location.getModule(), location.getPackage(), outputDirectory);
}
}
private static void createTable(String rootDir, String module, String packageName, String outputDirectory) throws IOException, ClassNotFoundException {
+ processConfigOptions(rootDir, module, packageName, optionsClass -> {
+ List<Tuple2<ConfigGroup, String>> tables = generateTablesForClass(optionsClass);
+ for (Tuple2<ConfigGroup, String> group : tables) {
+ String name;
+ if (group.f0 == null) {
+ Matcher matcher = CLASS_NAME_PATTERN.matcher(optionsClass.getSimpleName());
+ if (!matcher.matches()) {
+ throw new RuntimeException("Pattern did not match for " + optionsClass.getSimpleName() + '.');
+ }
+ name = matcher.group(CLASS_PREFIX_GROUP).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
+ } else {
+ name = group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
+ }
+
+ String outputFile = name + "_configuration.html";
+ Files.write(Paths.get(outputDirectory, outputFile), group.f1.getBytes(StandardCharsets.UTF_8));
+ }
+ });
+ }
+
+ private static void processConfigOptions(String rootDir, String module, String packageName, ThrowingConsumer<Class<?>, IOException> classConsumer) throws IOException, ClassNotFoundException {
Path configDir = Paths.get(rootDir, module, "src/main/java", packageName.replaceAll("\\.", "/"));
- Pattern p = Pattern.compile("(([a-zA-Z]*)(Options|Config|Parameters))\\.java");
try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir)) {
for (Path entry : stream) {
String fileName = entry.getFileName().toString();
- Matcher matcher = p.matcher(fileName);
+ Matcher matcher = CLASS_NAME_PATTERN.matcher(fileName);
if (!fileName.equals("ConfigOptions.java") && matcher.matches()) {
- Class<?> optionsClass = Class.forName(packageName + "." + matcher.group(1));
- List<Tuple2<ConfigGroup, String>> tables = generateTablesForClass(optionsClass);
- if (tables.size() > 0) {
- for (Tuple2<ConfigGroup, String> group : tables) {
-
- String name = group.f0 == null
- ? matcher.group(2).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase()
- : group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
-
- String outputFile = name + "_configuration.html";
- Files.write(Paths.get(outputDirectory, outputFile), group.f1.getBytes(StandardCharsets.UTF_8));
- }
- }
+ Class<?> optionsClass = Class.forName(packageName + '.' + matcher.group(CLASS_NAME_GROUP));
+ classConsumer.accept(optionsClass);
}
}
}
@@ -98,40 +120,41 @@ public class ConfigOptionsDocGenerator {
@VisibleForTesting
static List<Tuple2<ConfigGroup, String>> generateTablesForClass(Class<?> optionsClass) {
ConfigGroups configGroups = optionsClass.getAnnotation(ConfigGroups.class);
- List<Tuple2<ConfigGroup, String>> tables = new ArrayList<>();
- List<ConfigOption> allOptions = extractConfigOptions(optionsClass);
+ List<ConfigOption<?>> allOptions = extractConfigOptions(optionsClass);
+ List<Tuple2<ConfigGroup, String>> tables;
if (configGroups != null) {
+ tables = new ArrayList<>(configGroups.groups().length + 1);
Tree tree = new Tree(configGroups.groups(), allOptions);
for (ConfigGroup group : configGroups.groups()) {
- List<ConfigOption> configOptions = tree.findConfigOptions(group);
+ List<ConfigOption<?>> configOptions = tree.findConfigOptions(group);
sortOptions(configOptions);
tables.add(Tuple2.of(group, toHtmlTable(configOptions)));
}
- List<ConfigOption> configOptions = tree.getDefaultOptions();
+ List<ConfigOption<?>> configOptions = tree.getDefaultOptions();
sortOptions(configOptions);
tables.add(Tuple2.of(null, toHtmlTable(configOptions)));
} else {
sortOptions(allOptions);
- tables.add(Tuple2.of(null, toHtmlTable(allOptions)));
+ tables = Collections.singletonList(Tuple2.of(null, toHtmlTable(allOptions)));
}
return tables;
}
- private static List<ConfigOption> extractConfigOptions(Class<?> clazz) {
+ private static List<ConfigOption<?>> extractConfigOptions(Class<?> clazz) {
try {
- List<ConfigOption> configOptions = new ArrayList<>();
+ List<ConfigOption<?>> configOptions = new ArrayList<>(8);
Field[] fields = clazz.getFields();
for (Field field : fields) {
if (field.getType().equals(ConfigOption.class) && field.getAnnotation(Deprecated.class) == null) {
- configOptions.add((ConfigOption) field.get(null));
+ configOptions.add((ConfigOption<?>) field.get(null));
}
}
return configOptions;
} catch (Exception e) {
- throw new RuntimeException("Failed to extract config options from class " + clazz + ".", e);
+ throw new RuntimeException("Failed to extract config options from class " + clazz + '.', e);
}
}
@@ -142,7 +165,7 @@ public class ConfigOptionsDocGenerator {
* @param options list of options to include in this group
* @return string containing HTML formatted table
*/
- private static String toHtmlTable(final List<ConfigOption> options) {
+ private static String toHtmlTable(final List<ConfigOption<?>> options) {
StringBuilder htmlTable = new StringBuilder();
htmlTable.append("<table class=\"table table-bordered\">\n");
htmlTable.append(" <thead>\n");
@@ -154,7 +177,7 @@ public class ConfigOptionsDocGenerator {
htmlTable.append(" </thead>\n");
htmlTable.append(" <tbody>\n");
- for (ConfigOption option : options) {
+ for (ConfigOption<?> option : options) {
htmlTable.append(toHtmlString(option));
}
@@ -203,7 +226,7 @@ public class ConfigOptionsDocGenerator {
.replaceAll(">", ">");
}
- private static void sortOptions(List<ConfigOption> configOptions) {
+ private static void sortOptions(List<ConfigOption<?>> configOptions) {
configOptions.sort(Comparator.comparing(ConfigOption::key));
}
@@ -214,7 +237,7 @@ public class ConfigOptionsDocGenerator {
private static class Tree {
private final Node root = new Node();
- Tree(ConfigGroup[] groups, Collection<ConfigOption> options) {
+ Tree(ConfigGroup[] groups, Collection<ConfigOption<?>> options) {
// generate a tree based on all key prefixes
for (ConfigGroup group : groups) {
String[] keyComponents = group.keyPrefix().split("\\.");
@@ -232,12 +255,12 @@ public class ConfigOptionsDocGenerator {
}
}
- List<ConfigOption> findConfigOptions(ConfigGroup configGroup) {
+ List<ConfigOption<?>> findConfigOptions(ConfigGroup configGroup) {
Node groupRoot = findGroupRoot(configGroup.keyPrefix());
return groupRoot.getConfigOptions();
}
- List<ConfigOption> getDefaultOptions() {
+ List<ConfigOption<?>> getDefaultOptions() {
return root.getConfigOptions();
}
@@ -251,8 +274,8 @@ public class ConfigOptionsDocGenerator {
}
private static class Node {
- private final List<ConfigOption> configOptions = new ArrayList<>();
- private final Map<String, Node> children = new HashMap<>();
+ private final List<ConfigOption<?>> configOptions = new ArrayList<>(8);
+ private final Map<String, Node> children = new HashMap<>(8);
private boolean isGroupRoot = false;
private Node addChild(String keyComponent) {
@@ -272,7 +295,7 @@ public class ConfigOptionsDocGenerator {
return child;
}
- private void assignOption(ConfigOption option) {
+ private void assignOption(ConfigOption<?> option) {
configOptions.add(option);
}
@@ -284,7 +307,7 @@ public class ConfigOptionsDocGenerator {
this.isGroupRoot = true;
}
- private List<ConfigOption> getConfigOptions() {
+ private List<ConfigOption<?>> getConfigOptions() {
return configOptions;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/612e2d83/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java
----------------------------------------------------------------------
diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java
new file mode 100644
index 0000000..2fc11a9
--- /dev/null
+++ b/flink-docs/src/main/java/org/apache/flink/docs/configuration/OptionsClassLocation.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.docs.configuration;
+
+import org.apache.flink.configuration.ConfigOption;
+
+/**
+ * Simple descriptor for the location of a class containing {@link ConfigOption ConfigOptions}.
+ */
+class OptionsClassLocation {
+ private final String module;
+ private final String pckg;
+
+ OptionsClassLocation(String module, String pckg) {
+ this.module = module;
+ this.pckg = pckg;
+ }
+
+ public String getModule() {
+ return module;
+ }
+
+ public String getPackage() {
+ return pckg;
+ }
+}