You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2018/08/16 07:20:09 UTC
[flink] branch release-1.6 updated (4aa5d7f -> def4895)
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a change to branch release-1.6
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 4aa5d7f [FLINK-10101][mesos] Announce web UI url at Mesos master
new 3dccd07 [FLINK-9013] Allow formatting text as code in option description
new def4895 [FLINK-9013] Document yarn.containers.vcores only being effective when adapting YARN config
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../generated/yarn_config_configuration.html | 2 +-
docs/ops/deployment/yarn_setup.md | 4 ++--
.../configuration/description/Description.java | 11 +++++++++
.../flink/configuration/description/Formatter.java | 10 ++++++--
.../configuration/description/HtmlFormatter.java | 17 +++++++++++++-
.../configuration/description/TextElement.java | 27 +++++++++++++++++++++-
.../yarn/configuration/YarnConfigOptions.java | 13 ++++++++---
7 files changed, 74 insertions(+), 10 deletions(-)
[flink] 02/02: [FLINK-9013] Document yarn.containers.vcores only
being effective when adapting YARN config
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch release-1.6
in repository https://gitbox.apache.org/repos/asf/flink.git
commit def489554fc8e999a256ea2b1b0bd6e6f65345ff
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Fri Aug 10 13:06:37 2018 +0200
[FLINK-9013] Document yarn.containers.vcores only being effective when adapting YARN config
---
docs/_includes/generated/yarn_config_configuration.html | 2 +-
docs/ops/deployment/yarn_setup.md | 4 ++--
.../apache/flink/yarn/configuration/YarnConfigOptions.java | 13 ++++++++++---
3 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/docs/_includes/generated/yarn_config_configuration.html b/docs/_includes/generated/yarn_config_configuration.html
index fb16e53..bbe2549 100644
--- a/docs/_includes/generated/yarn_config_configuration.html
+++ b/docs/_includes/generated/yarn_config_configuration.html
@@ -30,7 +30,7 @@
<tr>
<td><h5>yarn.containers.vcores</h5></td>
<td style="word-wrap: break-word;">-1</td>
- <td>The number of virtual cores (vcores) per YARN container. By default, the number of vcores is set to the number of slots per TaskManager, if set, or to 1, otherwise.</td>
+ <td>The number of virtual cores (vcores) per YARN container. By default, the number of vcores is set to the number of slots per TaskManager, if set, or to 1, otherwise. In order for this parameter to be used your cluster must have CPU scheduling enabled. You can do this by setting the <span markdown="span">`org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler`</span>.</td>
</tr>
<tr>
<td><h5>yarn.heartbeat-delay</h5></td>
diff --git a/docs/ops/deployment/yarn_setup.md b/docs/ops/deployment/yarn_setup.md
index 9c255d2..18e693e 100644
--- a/docs/ops/deployment/yarn_setup.md
+++ b/docs/ops/deployment/yarn_setup.md
@@ -131,8 +131,8 @@ Once Flink is deployed in your YARN cluster, it will show you the connection det
Stop the YARN session by stopping the unix process (using CTRL+C) or by entering 'stop' into the client.
Flink on YARN will only start all requested containers if enough resources are available on the cluster. Most YARN schedulers account for the requested memory of the containers,
-some account also for the number of vcores. By default, the number of vcores is equal to the processing slots (`-s`) argument. The `yarn.containers.vcores` allows overwriting the
-number of vcores with a custom value.
+some account also for the number of vcores. By default, the number of vcores is equal to the processing slots (`-s`) argument. The [`yarn.containers.vcores`]({{ site.baseurl }}/ops/config.html#yarn-containers-vcores) allows overwriting the
+number of vcores with a custom value. In order for this parameter to work you should enable CPU scheduling in your cluster.
#### Detached YARN Session
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
index 255a6c7..b059475 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
@@ -19,8 +19,10 @@
package org.apache.flink.yarn.configuration;
import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.description.Description;
import static org.apache.flink.configuration.ConfigOptions.key;
+import static org.apache.flink.configuration.description.TextElement.code;
/**
* This class holds configuration constants used by Flink's YARN runners.
@@ -63,9 +65,14 @@ public class YarnConfigOptions {
*/
public static final ConfigOption<Integer> VCORES =
key("yarn.containers.vcores")
- .defaultValue(-1)
- .withDescription("The number of virtual cores (vcores) per YARN container. By default, the number of vcores" +
- " is set to the number of slots per TaskManager, if set, or to 1, otherwise.");
+ .defaultValue(-1)
+ .withDescription(Description.builder().text(
+ "The number of virtual cores (vcores) per YARN container. By default, the number of vcores" +
+ " is set to the number of slots per TaskManager, if set, or to 1, otherwise. In order for this" +
+ " parameter to be used your cluster must have CPU scheduling enabled. You can do this by setting" +
+ " the %s.",
+ code("org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"))
+ .build());
/**
* The maximum number of failed YARN containers before entirely stopping
[flink] 01/02: [FLINK-9013] Allow formatting text as code in option
description
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch release-1.6
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3dccd071f31320110a720bdca05d1a974d37376c
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Aug 9 17:10:38 2018 +0200
[FLINK-9013] Allow formatting text as code in option description
---
.../configuration/description/Description.java | 11 +++++++++
.../flink/configuration/description/Formatter.java | 10 ++++++--
.../configuration/description/HtmlFormatter.java | 17 +++++++++++++-
.../configuration/description/TextElement.java | 27 +++++++++++++++++++++-
4 files changed, 61 insertions(+), 4 deletions(-)
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/description/Description.java b/flink-core/src/main/java/org/apache/flink/configuration/description/Description.java
index 25a6a64..c00890d 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/description/Description.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/description/Description.java
@@ -79,6 +79,17 @@ public class Description {
}
/**
+ * Block of description add.
+ *
+ * @param block block of description to add
+ * @return block of description
+ */
+ public DescriptionBuilder add(BlockElement block) {
+ blocks.add(block);
+ return this;
+ }
+
+ /**
* Creates a line break in the description.
*/
public DescriptionBuilder linebreak() {
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/description/Formatter.java b/flink-core/src/main/java/org/apache/flink/configuration/description/Formatter.java
index d3fcf40..fdf7db6 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/description/Formatter.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/description/Formatter.java
@@ -18,6 +18,8 @@
package org.apache.flink.configuration.description;
+import java.util.EnumSet;
+
/**
* Allows providing multiple formatters for the description. E.g. Html formatter, Markdown formatter etc.
*/
@@ -49,7 +51,7 @@ public abstract class Formatter {
return formatter.finalizeFormatting();
}
).toArray(String[]::new);
- formatText(state, escapeFormatPlaceholder(element.getFormat()), inlineElements);
+ formatText(state, escapeFormatPlaceholder(element.getFormat()), inlineElements, element.getStyles());
}
public void format(LineBreakElement element) {
@@ -76,7 +78,11 @@ public abstract class Formatter {
protected abstract void formatLineBreak(StringBuilder state);
- protected abstract void formatText(StringBuilder state, String format, String[] elements);
+ protected abstract void formatText(
+ StringBuilder state,
+ String format,
+ String[] elements,
+ EnumSet<TextElement.TextStyle> styles);
protected abstract void formatList(StringBuilder state, String[] entries);
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java b/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java
index 72531ab..a475303 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java
@@ -18,6 +18,8 @@
package org.apache.flink.configuration.description;
+import java.util.EnumSet;
+
/**
* Formatter that transforms {@link Description} into Html representation.
*/
@@ -34,9 +36,22 @@ public class HtmlFormatter extends Formatter {
}
@Override
- protected void formatText(StringBuilder state, String format, String[] elements) {
+ protected void formatText(
+ StringBuilder state,
+ String format,
+ String[] elements,
+ EnumSet<TextElement.TextStyle> styles) {
String escapedFormat = escapeCharacters(format);
+
+ String prefix = "";
+ String suffix = "";
+ if (styles.contains(TextElement.TextStyle.CODE)) {
+ prefix = "<span markdown=\"span\">`";
+ suffix = "`</span>";
+ }
+ state.append(prefix);
state.append(String.format(escapedFormat, elements));
+ state.append(suffix);
}
@Override
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/description/TextElement.java b/flink-core/src/main/java/org/apache/flink/configuration/description/TextElement.java
index aeb1d7a..80bdfcf 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/description/TextElement.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/description/TextElement.java
@@ -20,6 +20,7 @@ package org.apache.flink.configuration.description;
import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.List;
/**
@@ -28,6 +29,7 @@ import java.util.List;
public class TextElement implements BlockElement, InlineElement {
private final String format;
private final List<InlineElement> elements;
+ private final EnumSet<TextStyle> textStyles = EnumSet.noneOf(TextStyle.class);
/**
* Creates a block of text with placeholders ("%s") that will be replaced with proper string representation of
@@ -35,7 +37,7 @@ public class TextElement implements BlockElement, InlineElement {
*
* <p>{@code text("This is a text with a link %s", link("https://somepage", "to here"))}
*
- * @param format text with placeholders for elements
+ * @param format text with placeholders for elements
* @param elements elements to be put in the text
* @return block of text
*/
@@ -53,6 +55,18 @@ public class TextElement implements BlockElement, InlineElement {
return new TextElement(text, Collections.emptyList());
}
+ /**
+ * Creates a block of text formatted as code.
+ *
+ * @param text a block of text that will be formatted as code
+ * @return block of text formatted as code
+ */
+ public static TextElement code(String text) {
+ TextElement element = text(text);
+ element.textStyles.add(TextStyle.CODE);
+ return element;
+ }
+
public String getFormat() {
return format;
}
@@ -61,6 +75,10 @@ public class TextElement implements BlockElement, InlineElement {
return elements;
}
+ public EnumSet<TextStyle> getStyles() {
+ return textStyles;
+ }
+
private TextElement(String format, List<InlineElement> elements) {
this.format = format;
this.elements = elements;
@@ -70,4 +88,11 @@ public class TextElement implements BlockElement, InlineElement {
public void format(Formatter formatter) {
formatter.format(this);
}
+
+ /**
+ * Styles that can be applied to {@link TextElement} e.g. code, bold etc.
+ */
+ public enum TextStyle {
+ CODE
+ }
}