You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by kk...@apache.org on 2019/10/25 14:47:57 UTC
[flink] 07/08: [hotfix] Do not expose Yarn dynamic options
This is an automated email from the ASF dual-hosted git repository.
kkloudas pushed a commit to branch to-merge
in repository https://gitbox.apache.org/repos/asf/flink.git
commit d457becca501e1bc70bc0e569754ea87bd8c240a
Author: Kostas Kloudas <kk...@gmail.com>
AuthorDate: Fri Oct 25 11:46:26 2019 +0200
[hotfix] Do not expose Yarn dynamic options
---
.../generated/yarn_config_configuration.html | 5 ---
.../flink/yarn/YarnClusterClientFactory.java | 3 +-
.../apache/flink/yarn/cli/FlinkYarnSessionCli.java | 3 +-
.../yarn/configuration/YarnConfigOptions.java | 5 ---
.../configuration/YarnConfigOptionsInternal.java | 37 ++++++++++++++++++++++
5 files changed, 41 insertions(+), 12 deletions(-)
diff --git a/docs/_includes/generated/yarn_config_configuration.html b/docs/_includes/generated/yarn_config_configuration.html
index 6cf3945..98655bf 100644
--- a/docs/_includes/generated/yarn_config_configuration.html
+++ b/docs/_includes/generated/yarn_config_configuration.html
@@ -8,11 +8,6 @@
</thead>
<tbody>
<tr>
- <td><h5>$internal.yarn.dynamic-properties</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>**DO NOT USE** Specify YARN dynamic properties.</td>
- </tr>
- <tr>
<td><h5>yarn.application-attempt-failures-validity-interval</h5></td>
<td style="word-wrap: break-word;">10000</td>
<td>Time window in milliseconds which defines the number of application attempt failures when restarting the AM. Failures which fall outside of this window are not being considered. Set this value to -1 in order to count globally. See <a href="https://hortonworks.com/blog/apache-hadoop-yarn-hdp-2-2-fault-tolerance-features-long-running-services/">here</a> for more information.</td>
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClientFactory.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClientFactory.java
index a0d9ab2..a954be8 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClientFactory.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClientFactory.java
@@ -28,6 +28,7 @@ import org.apache.flink.configuration.HighAvailabilityOptions;
import org.apache.flink.configuration.TaskManagerOptions;
import org.apache.flink.yarn.cli.YarnConfigUtils;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.configuration.YarnConfigOptionsInternal;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -78,7 +79,7 @@ public class YarnClusterClientFactory implements ClusterClientFactory<Applicatio
.ifPresent(yarnClusterDescriptor::addShipFiles);
handleConfigOption(configuration, YarnConfigOptions.APPLICATION_QUEUE, yarnClusterDescriptor::setQueue);
- handleConfigOption(configuration, YarnConfigOptions.DYNAMIC_PROPERTIES, yarnClusterDescriptor::setDynamicPropertiesEncoded);
+ handleConfigOption(configuration, YarnConfigOptionsInternal.DYNAMIC_PROPERTIES, yarnClusterDescriptor::setDynamicPropertiesEncoded);
handleConfigOption(configuration, YarnConfigOptions.APPLICATION_NAME, yarnClusterDescriptor::setName);
handleConfigOption(configuration, YarnConfigOptions.APPLICATION_TYPE, yarnClusterDescriptor::setApplicationType);
handleConfigOption(configuration, YarnConfigOptions.NODE_LABEL, yarnClusterDescriptor::setNodeLabel);
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
index e301a74..0efa610 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
@@ -44,6 +44,7 @@ import org.apache.flink.util.ShutdownHookUtil;
import org.apache.flink.yarn.YarnClusterClientFactory;
import org.apache.flink.yarn.YarnClusterDescriptor;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.configuration.YarnConfigOptionsInternal;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
@@ -413,7 +414,7 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine {
}
final String dynamicPropertiesEncoded = encodeDynamicProperties(commandLine);
- configuration.setString(YarnConfigOptions.DYNAMIC_PROPERTIES, dynamicPropertiesEncoded);
+ configuration.setString(YarnConfigOptionsInternal.DYNAMIC_PROPERTIES, dynamicPropertiesEncoded);
final boolean detached = commandLine.hasOption(YARN_DETACHED_OPTION.getOpt()) || commandLine.hasOption(DETACHED_OPTION.getOpt());
configuration.setBoolean(DeploymentOptions.ATTACHED, !detached);
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
index 8ef7293..021552d 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
@@ -211,11 +211,6 @@ public class YarnConfigOptions {
.noDefaultValue()
.withDescription("The location of the log config file, e.g. the path to your log4j.properties for log4j.");
- public static final ConfigOption<String> DYNAMIC_PROPERTIES =
- key("$internal.yarn.dynamic-properties")
- .noDefaultValue()
- .withDescription("**DO NOT USE** Specify YARN dynamic properties.");
-
public static final ConfigOption<String> SHIP_DIRECTORIES =
key("yarn.ship-directories")
.noDefaultValue()
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptionsInternal.java b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptionsInternal.java
new file mode 100644
index 0000000..4910809
--- /dev/null
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptionsInternal.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn.configuration;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.ConfigOption;
+
+import static org.apache.flink.configuration.ConfigOptions.key;
+
+/**
+ * Yarn configuration options that are not meant to be set by the user.
+ */
+@Internal
+public class YarnConfigOptionsInternal {
+
+ public static final ConfigOption<String> DYNAMIC_PROPERTIES =
+ key("$internal.yarn.dynamic-properties")
+ .noDefaultValue()
+ .withDescription("**DO NOT USE** Specify YARN dynamic properties.");
+
+}