You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2018/07/25 08:57:33 UTC
[1/2] flink git commit: [FLINK-6222] Update config.sh to look for a
Hadoop and/or YARN configuration directories
Repository: flink
Updated Branches:
refs/heads/master cc5c85547 -> 4c4e3d66c
[FLINK-6222] Update config.sh to look for a Hadoop and/or YARN configuration directories
This just adds to the config.sh to allow setting YARN_CONF_DIR and HADOOP_CONF_DIR
with the Flink configuration file.
This closes #6388
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/81ff3c45
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/81ff3c45
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/81ff3c45
Branch: refs/heads/master
Commit: 81ff3c45004cd6e3cc5b0f75dc567f5f57d1f1ed
Parents: cc5c855
Author: Craig Foster <fo...@amazon.com>
Authored: Fri Mar 31 09:41:21 2017 -0700
Committer: Dawid Wysakowicz <dw...@apache.org>
Committed: Wed Jul 25 10:55:50 2018 +0200
----------------------------------------------------------------------
flink-dist/src/main/flink-bin/bin/config.sh | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/81ff3c45/flink-dist/src/main/flink-bin/bin/config.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh
index 82b27c1..c74dfc0 100755
--- a/flink-dist/src/main/flink-bin/bin/config.sh
+++ b/flink-dist/src/main/flink-bin/bin/config.sh
@@ -96,6 +96,8 @@ DEFAULT_ENV_JAVA_OPTS="" # Optional JVM args
DEFAULT_ENV_JAVA_OPTS_JM="" # Optional JVM args (JobManager)
DEFAULT_ENV_JAVA_OPTS_TM="" # Optional JVM args (TaskManager)
DEFAULT_ENV_SSH_OPTS="" # Optional SSH parameters running in cluster mode
+DEFAULT_YARN_CONF_DIR="" # YARN Configuration Directory, if necessary
+DEFAULT_HADOOP_CONF_DIR="" # Hadoop Configuration Directory, if necessary
########################################################################################################################
# CONFIG KEYS: The default values can be overwritten by the following keys in conf/flink-conf.yaml
@@ -120,6 +122,8 @@ KEY_TASKM_COMPUTE_NUMA="taskmanager.compute.numa"
KEY_ENV_PID_DIR="env.pid.dir"
KEY_ENV_LOG_DIR="env.log.dir"
KEY_ENV_LOG_MAX="env.log.max"
+KEY_ENV_YARN_CONF_DIR="env.yarn.conf.dir"
+KEY_ENV_HADOOP_CONF_DIR="env.hadoop.conf.dir"
KEY_ENV_JAVA_HOME="env.java.home"
KEY_ENV_JAVA_OPTS="env.java.opts"
KEY_ENV_JAVA_OPTS_JM="env.java.opts.jobmanager"
@@ -450,6 +454,14 @@ if [ -z "${FLINK_LOG_DIR}" ]; then
FLINK_LOG_DIR=$(readFromConfig ${KEY_ENV_LOG_DIR} "${DEFAULT_FLINK_LOG_DIR}" "${YAML_CONF}")
fi
+if [ -z "${YARN_CONF_DIR}" ]; then
+ YARN_CONF_DIR=$(readFromConfig ${KEY_ENV_YARN_CONF_DIR} "${DEFAULT_YARN_CONF_DIR}" "${YAML_CONF}")
+fi
+
+if [ -z "${HADOOP_CONF_DIR}" ]; then
+ HADOOP_CONF_DIR=$(readFromConfig ${KEY_ENV_HADOOP_CONF_DIR} "${DEFAULT_HADOOP_CONF_DIR}" "${YAML_CONF}")
+fi
+
if [ -z "${FLINK_PID_DIR}" ]; then
FLINK_PID_DIR=$(readFromConfig ${KEY_ENV_PID_DIR} "${DEFAULT_ENV_PID_DIR}" "${YAML_CONF}")
fi
[2/2] flink git commit: [FLINK-6222] Documented YARN and HADOOP conf
dir options.
Posted by dw...@apache.org.
[FLINK-6222] Documented YARN and HADOOP conf dir options.
This closes #6388
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4c4e3d66
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4c4e3d66
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4c4e3d66
Branch: refs/heads/master
Commit: 4c4e3d66c9635e523c8ab1539f635ef1dbb24e61
Parents: 81ff3c4
Author: Dawid Wysakowicz <dw...@apache.org>
Authored: Tue Jul 24 09:03:44 2018 +0200
Committer: Dawid Wysakowicz <dw...@apache.org>
Committed: Wed Jul 25 10:56:02 2018 +0200
----------------------------------------------------------------------
.../generated/environment_configuration.html | 10 +++++++++
.../apache/flink/configuration/CoreOptions.java | 22 ++++++++++++++++++++
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4c4e3d66/docs/_includes/generated/environment_configuration.html
----------------------------------------------------------------------
diff --git a/docs/_includes/generated/environment_configuration.html b/docs/_includes/generated/environment_configuration.html
index a56a8c2..7db585d 100644
--- a/docs/_includes/generated/environment_configuration.html
+++ b/docs/_includes/generated/environment_configuration.html
@@ -8,6 +8,11 @@
</thead>
<tbody>
<tr>
+ <td><h5>env.hadoop.conf.dir</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>Path to hadoop configuration directory. It is required to read HDFS and/or YARN configuration. You can also set it via environment variable.</td>
+ </tr>
+ <tr>
<td><h5>env.java.opts</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td></td>
@@ -37,5 +42,10 @@
<td style="word-wrap: break-word;">(none)</td>
<td>Additional command line options passed to SSH clients when starting or stopping JobManager, TaskManager, and Zookeeper services (start-cluster.sh, stop-cluster.sh, start-zookeeper-quorum.sh, stop-zookeeper-quorum.sh).</td>
</tr>
+ <tr>
+ <td><h5>env.yarn.conf.dir</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>Path to yarn configuration directory. It is required to run flink on YARN. You can also set it via environment variable.</td>
+ </tr>
</tbody>
</table>
http://git-wip-us.apache.org/repos/asf/flink/blob/4c4e3d66/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
index 16c9b54..f410204 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
@@ -173,6 +173,28 @@ public class CoreOptions {
" TaskManager, and Zookeeper services (start-cluster.sh, stop-cluster.sh, start-zookeeper-quorum.sh," +
" stop-zookeeper-quorum.sh).");
+ /**
+ * This options is here only for documentation generation, it is only
+ * evaluated in the shell scripts.
+ */
+ @SuppressWarnings("unused")
+ public static final ConfigOption<String> FLINK_HADOOP_CONF_DIR = ConfigOptions
+ .key("env.hadoop.conf.dir")
+ .noDefaultValue()
+ .withDescription("Path to hadoop configuration directory. It is required to read HDFS and/or YARN" +
+ " configuration. You can also set it via environment variable.");
+
+ /**
+ * This options is here only for documentation generation, it is only
+ * evaluated in the shell scripts.
+ */
+ @SuppressWarnings("unused")
+ public static final ConfigOption<String> FLINK_YARN_CONF_DIR = ConfigOptions
+ .key("env.yarn.conf.dir")
+ .noDefaultValue()
+ .withDescription("Path to yarn configuration directory. It is required to run flink on YARN. You can also" +
+ " set it via environment variable.");
+
// ------------------------------------------------------------------------
// generic io
// ------------------------------------------------------------------------