You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/05/02 13:19:45 UTC
[1/7] flink git commit: [FLINK-9216][Streaming] Fix comparator
violation
Repository: flink
Updated Branches:
refs/heads/release-1.5 41e2b3e17 -> dd5639164
[FLINK-9216][Streaming] Fix comparator violation
This closes #5878.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/05c6ef1e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/05c6ef1e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/05c6ef1e
Branch: refs/heads/release-1.5
Commit: 05c6ef1efede6a3f51eb79103ae0ff392dda7b07
Parents: 547152e
Author: Xpray <le...@gmail.com>
Authored: Mon Apr 23 15:37:39 2018 +0800
Committer: zentol <ch...@apache.org>
Committed: Wed May 2 15:18:59 2018 +0200
----------------------------------------------------------------------
.../streaming/api/graph/JSONGenerator.java | 12 ++++++----
.../api/graph/StreamGraphGeneratorTest.java | 24 ++++++++++++++++++++
2 files changed, 31 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/05c6ef1e/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java
index 263e0aa..3f82cf3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java
@@ -61,14 +61,16 @@ public class JSONGenerator {
List<Integer> operatorIDs = new ArrayList<Integer>(streamGraph.getVertexIDs());
Collections.sort(operatorIDs, new Comparator<Integer>() {
@Override
- public int compare(Integer o1, Integer o2) {
+ public int compare(Integer idOne, Integer idTwo) {
+ boolean isIdOneSinkId = streamGraph.getSinkIDs().contains(idOne);
+ boolean isIdTwoSinkId = streamGraph.getSinkIDs().contains(idTwo);
// put sinks at the back
- if (streamGraph.getSinkIDs().contains(o1)) {
+ if (isIdOneSinkId == isIdTwoSinkId) {
+ return idOne.compareTo(idTwo);
+ } else if (isIdOneSinkId) {
return 1;
- } else if (streamGraph.getSinkIDs().contains(o2)) {
- return -1;
} else {
- return o1 - o2;
+ return -1;
}
}
});
http://git-wip-us.apache.org/repos/asf/flink/blob/05c6ef1e/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
index 8149d24..d10fb3c 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;
import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
+import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.Output;
@@ -413,6 +414,29 @@ public class StreamGraphGeneratorTest {
StreamPartitioner<?> streamPartitioner2 = keyedResultNode.getInEdges().get(1).getPartitioner();
}
+ /**
+ * Tests that the json generated by JSONGenerator shall meet with 2 requirements:
+ * 1. sink nodes are at the back
+ * 2. if both two nodes are sink nodes or neither of them is sink node, then sort by its id.
+ */
+ @Test
+ public void testSinkIdComparison() {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ DataStream<Integer> source = env.fromElements(1, 2, 3);
+ for (int i = 0; i < 32; i++) {
+ if (i % 2 == 0) {
+ source.addSink(new SinkFunction<Integer>() {
+ @Override
+ public void invoke(Integer value) throws Exception {}
+ });
+ } else {
+ source.map(x -> x + 1);
+ }
+ }
+ // IllegalArgumentException will be thrown without FLINK-9216
+ env.getStreamGraph().getStreamingPlanAsJSON();
+ }
+
private static class OutputTypeConfigurableOperationWithTwoInputs
extends AbstractStreamOperator<Integer>
implements TwoInputStreamOperator<Integer, Integer, Integer>, OutputTypeConfigurable<Integer> {
[3/7] flink git commit: [FLINK-9249][build] Add convenience profile
for skipping non-essential plugins
Posted by ch...@apache.org.
[FLINK-9249][build] Add convenience profile for skipping non-essential plugins
This closes #5904.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/45354899
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/45354899
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/45354899
Branch: refs/heads/release-1.5
Commit: 45354899cef40ed6ce8a38119ed87003b5286acc
Parents: 05c6ef1
Author: zentol <ch...@apache.org>
Authored: Tue Apr 24 13:13:39 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed May 2 15:19:30 2018 +0200
----------------------------------------------------------------------
pom.xml | 57 +++++++++++++++++++++++++++++++++++++++
tools/travis_mvn_watchdog.sh | 20 +++++++-------
2 files changed, 67 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/45354899/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 49a0a56..ce2a77e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -637,6 +637,63 @@ under the License.
<profiles>
<profile>
+ <id>fast</id>
+ <activation>
+ <property>
+ <name>fast</name>
+ </property>
+ </activation>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>com.github.siom79.japicmp</groupId>
+ <artifactId>japicmp-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
+
+ <profile>
<id>legacyCode</id>
<activation>
<property>
http://git-wip-us.apache.org/repos/asf/flink/blob/45354899/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index eacecc6..e08f0cc 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -117,26 +117,26 @@ case $TEST in
(core)
MVN_COMPILE_MODULES="-pl $MODULES_CORE -am"
MVN_TEST_MODULES="-pl $MODULES_CORE"
- MVN_COMPILE_OPTIONS="-Dcheckstyle.skip=true -Djapicmp.skip=true -Drat.skip=true"
- MVN_TEST_OPTIONS="-Dcheckstyle.skip=true"
+ MVN_COMPILE_OPTIONS="-Dfast"
+ MVN_TEST_OPTIONS="-Dfast"
;;
(libraries)
MVN_COMPILE_MODULES="-pl $MODULES_LIBRARIES -am"
MVN_TEST_MODULES="-pl $MODULES_LIBRARIES"
- MVN_COMPILE_OPTIONS="-Dcheckstyle.skip=true -Djapicmp.skip=true -Drat.skip=true"
- MVN_TEST_OPTIONS="-Dcheckstyle.skip=true"
+ MVN_COMPILE_OPTIONS="-Dfast"
+ MVN_TEST_OPTIONS="-Dfast"
;;
(connectors)
MVN_COMPILE_MODULES="-pl $MODULES_CONNECTORS -am"
MVN_TEST_MODULES="-pl $MODULES_CONNECTORS"
- MVN_COMPILE_OPTIONS="-Dcheckstyle.skip=true -Djapicmp.skip=true -Drat.skip=true"
- MVN_TEST_OPTIONS="-Dcheckstyle.skip=true"
+ MVN_COMPILE_OPTIONS="-Dfast"
+ MVN_TEST_OPTIONS="-Dfast"
;;
(tests)
MVN_COMPILE_MODULES="-pl $MODULES_TESTS -am"
MVN_TEST_MODULES="-pl $MODULES_TESTS"
- MVN_COMPILE_OPTIONS="-Dcheckstyle.skip=true -Djapicmp.skip=true -Drat.skip=true"
- MVN_TEST_OPTIONS="-Dcheckstyle.skip=true"
+ MVN_COMPILE_OPTIONS="-Dfast"
+ MVN_TEST_OPTIONS="-Dfast"
;;
(misc)
NEGATED_CORE=\!${MODULES_CORE//,/,\!}
@@ -147,7 +147,7 @@ case $TEST in
MVN_COMPILE_MODULES=""
MVN_TEST_MODULES="-pl $NEGATED_CORE,$NEGATED_LIBRARIES,$NEGATED_CONNECTORS,$NEGATED_TESTS"
MVN_COMPILE_OPTIONS=""
- MVN_TEST_OPTIONS="-Dcheckstyle.skip=true"
+ MVN_TEST_OPTIONS="-Dfast"
;;
esac
@@ -158,7 +158,7 @@ esac
# -nsu option forbids downloading snapshot artifacts. The only snapshot artifacts we depend are from
# Flink, which however should all be built locally. see FLINK-7230
MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configuration=file://$LOG4J_PROPERTIES -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
-MVN_COMMON_OPTIONS="-nsu -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true -B $MVN_LOGGING_OPTIONS"
+MVN_COMMON_OPTIONS="-nsu -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -B $MVN_LOGGING_OPTIONS"
MVN_COMPILE_OPTIONS="$MVN_COMPILE_OPTIONS -DskipTests"
MVN_COMPILE="mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES clean install"
[2/7] flink git commit: [FLINK-9213] Revert checkpoint details URL
from '/jobs/:jobid/checkpoints/:checkpointid' to
'/jobs/:jobid/checkpoints/details/:checkpointid' as it was in 1.5
Posted by ch...@apache.org.
[FLINK-9213] Revert checkpoint details URL from '/jobs/:jobid/checkpoints/:checkpointid' to '/jobs/:jobid/checkpoints/details/:checkpointid' as it was in 1.5
This closes #5906.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/547152ea
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/547152ea
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/547152ea
Branch: refs/heads/release-1.5
Commit: 547152eadd54f2b0d827b9fd53bf2a0d0a0c893e
Parents: 41e2b3e
Author: Andrey Zagrebin <an...@linkresearchtools.org>
Authored: Tue Apr 24 13:26:40 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed May 2 15:18:59 2018 +0200
----------------------------------------------------------------------
docs/_includes/generated/rest_dispatcher.html | 202 +++++++++----------
.../CheckpointStatisticDetailsHeaders.java | 2 +-
2 files changed, 102 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/547152ea/docs/_includes/generated/rest_dispatcher.html
----------------------------------------------------------------------
diff --git a/docs/_includes/generated/rest_dispatcher.html b/docs/_includes/generated/rest_dispatcher.html
index 8ac36e1..98c9d42 100644
--- a/docs/_includes/generated/rest_dispatcher.html
+++ b/docs/_includes/generated/rest_dispatcher.html
@@ -1094,7 +1094,7 @@
<table class="table table-bordered">
<tbody>
<tr>
- <td class="text-left" colspan="2"><strong>/jobs/:jobid/checkpoints/details/:checkpointid/subtasks/:vertexid</strong></td>
+ <td class="text-left" colspan="2"><strong>/jobs/:jobid/checkpoints/details/:checkpointid</strong></td>
</tr>
<tr>
<td class="text-left" style="width: 20%">Verb: <code>GET</code></td>
@@ -1111,14 +1111,13 @@
<ul>
<li><code>jobid</code> - description</li>
<li><code>checkpointid</code> - description</li>
-<li><code>vertexid</code> - description</li>
</ul>
</td>
</tr>
<tr>
<td colspan="2">
- <button data-toggle="collapse" data-target="#206551710">Request</button>
- <div id="206551710" class="collapse">
+ <button data-toggle="collapse" data-target="#2082812513">Request</button>
+ <div id="2082812513" class="collapse">
<pre>
<code>
{} </code>
@@ -1128,13 +1127,13 @@
</tr>
<tr>
<td colspan="2">
- <button data-toggle="collapse" data-target="#147094768">Response</button>
- <div id="147094768" class="collapse">
+ <button data-toggle="collapse" data-target="#59893771">Response</button>
+ <div id="59893771" class="collapse">
<pre>
<code>
{
"type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics",
"properties" : {
"id" : {
"type" : "integer"
@@ -1143,6 +1142,12 @@
"type" : "string",
"enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
},
+ "is_savepoint" : {
+ "type" : "boolean"
+ },
+ "trigger_timestamp" : {
+ "type" : "integer"
+ },
"latest_ack_timestamp" : {
"type" : "integer"
},
@@ -1161,70 +1166,36 @@
"num_acknowledged_subtasks" : {
"type" : "integer"
},
- "summary" : {
+ "tasks" : {
"type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:Summary",
- "properties" : {
- "state_size" : {
- "type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics",
- "properties" : {
- "min" : {
- "type" : "integer"
- },
- "max" : {
- "type" : "integer"
- },
- "avg" : {
- "type" : "integer"
- }
- }
- },
- "end_to_end_duration" : {
- "type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
- },
- "checkpoint_duration" : {
- "type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:CheckpointDuration",
- "properties" : {
- "sync" : {
- "type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
- },
- "async" : {
- "type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
- }
- }
- },
- "alignment" : {
- "type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:CheckpointAlignment",
- "properties" : {
- "buffered" : {
- "type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
- },
- "duration" : {
- "type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
- }
- }
- }
- }
- },
- "subtasks" : {
- "type" : "array",
- "items" : {
+ "additionalProperties" : {
"type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:SubtaskCheckpointStatistics",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics",
"properties" : {
- "index" : {
+ "id" : {
"type" : "integer"
},
"status" : {
- "type" : "string"
+ "type" : "string",
+ "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
+ },
+ "latest_ack_timestamp" : {
+ "type" : "integer"
+ },
+ "state_size" : {
+ "type" : "integer"
+ },
+ "end_to_end_duration" : {
+ "type" : "integer"
+ },
+ "alignment_buffered" : {
+ "type" : "integer"
+ },
+ "num_subtasks" : {
+ "type" : "integer"
+ },
+ "num_acknowledged_subtasks" : {
+ "type" : "integer"
}
}
}
@@ -1240,7 +1211,7 @@
<table class="table table-bordered">
<tbody>
<tr>
- <td class="text-left" colspan="2"><strong>/jobs/:jobid/checkpoints/:checkpointid</strong></td>
+ <td class="text-left" colspan="2"><strong>/jobs/:jobid/checkpoints/details/:checkpointid/subtasks/:vertexid</strong></td>
</tr>
<tr>
<td class="text-left" style="width: 20%">Verb: <code>GET</code></td>
@@ -1257,13 +1228,14 @@
<ul>
<li><code>jobid</code> - description</li>
<li><code>checkpointid</code> - description</li>
+<li><code>vertexid</code> - description</li>
</ul>
</td>
</tr>
<tr>
<td colspan="2">
- <button data-toggle="collapse" data-target="#-968188818">Request</button>
- <div id="-968188818" class="collapse">
+ <button data-toggle="collapse" data-target="#206551710">Request</button>
+ <div id="206551710" class="collapse">
<pre>
<code>
{} </code>
@@ -1273,13 +1245,13 @@
</tr>
<tr>
<td colspan="2">
- <button data-toggle="collapse" data-target="#1303859736">Response</button>
- <div id="1303859736" class="collapse">
+ <button data-toggle="collapse" data-target="#147094768">Response</button>
+ <div id="147094768" class="collapse">
<pre>
<code>
{
"type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointStatistics",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails",
"properties" : {
"id" : {
"type" : "integer"
@@ -1288,12 +1260,6 @@
"type" : "string",
"enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
},
- "is_savepoint" : {
- "type" : "boolean"
- },
- "trigger_timestamp" : {
- "type" : "integer"
- },
"latest_ack_timestamp" : {
"type" : "integer"
},
@@ -1312,36 +1278,70 @@
"num_acknowledged_subtasks" : {
"type" : "integer"
},
- "tasks" : {
+ "summary" : {
"type" : "object",
- "additionalProperties" : {
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:Summary",
+ "properties" : {
+ "state_size" : {
+ "type" : "object",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics",
+ "properties" : {
+ "min" : {
+ "type" : "integer"
+ },
+ "max" : {
+ "type" : "integer"
+ },
+ "avg" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "end_to_end_duration" : {
+ "type" : "object",
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ },
+ "checkpoint_duration" : {
+ "type" : "object",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:CheckpointDuration",
+ "properties" : {
+ "sync" : {
+ "type" : "object",
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ },
+ "async" : {
+ "type" : "object",
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ }
+ }
+ },
+ "alignment" : {
+ "type" : "object",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatisticsWithSubtaskDetails:CheckpointAlignment",
+ "properties" : {
+ "buffered" : {
+ "type" : "object",
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ },
+ "duration" : {
+ "type" : "object",
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ }
+ }
+ }
+ }
+ },
+ "subtasks" : {
+ "type" : "array",
+ "items" : {
"type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:TaskCheckpointStatistics",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:SubtaskCheckpointStatistics",
"properties" : {
- "id" : {
+ "index" : {
"type" : "integer"
},
"status" : {
- "type" : "string",
- "enum" : [ "IN_PROGRESS", "COMPLETED", "FAILED" ]
- },
- "latest_ack_timestamp" : {
- "type" : "integer"
- },
- "state_size" : {
- "type" : "integer"
- },
- "end_to_end_duration" : {
- "type" : "integer"
- },
- "alignment_buffered" : {
- "type" : "integer"
- },
- "num_subtasks" : {
- "type" : "integer"
- },
- "num_acknowledged_subtasks" : {
- "type" : "integer"
+ "type" : "string"
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/547152ea/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointStatisticDetailsHeaders.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointStatisticDetailsHeaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointStatisticDetailsHeaders.java
index 3d7ba2b..efdef40 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointStatisticDetailsHeaders.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointStatisticDetailsHeaders.java
@@ -32,7 +32,7 @@ public class CheckpointStatisticDetailsHeaders implements MessageHeaders<EmptyRe
private static final CheckpointStatisticDetailsHeaders INSTANCE = new CheckpointStatisticDetailsHeaders();
- public static final String URL = "/jobs/:jobid/checkpoints/:checkpointid";
+ public static final String URL = "/jobs/:jobid/checkpoints/details/:checkpointid";
private CheckpointStatisticDetailsHeaders() {}
[4/7] flink git commit: [FLINK-9211][REST] JarRunHandler submits job
to Dispatcher via RPC
Posted by ch...@apache.org.
[FLINK-9211][REST] JarRunHandler submits job to Dispatcher via RPC
This closes #5903.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f4bef6fc
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f4bef6fc
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f4bef6fc
Branch: refs/heads/release-1.5
Commit: f4bef6fc51c297b372b95c9fc461fbf6c495a1fd
Parents: d83e19f
Author: zentol <ch...@apache.org>
Authored: Mon Apr 23 12:35:51 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed May 2 15:19:31 2018 +0200
----------------------------------------------------------------------
.../webmonitor/WebSubmissionExtension.java | 23 +---
.../webmonitor/handlers/JarRunHandler.java | 65 ++++++++---
.../handlers/JarRunMessageParameters.java | 12 +--
.../webmonitor/handlers/JarRunHandlerTest.java | 108 +++++++++++++++++++
.../runtime/webmonitor/WebMonitorUtils.java | 3 +-
5 files changed, 171 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/f4bef6fc/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java
index bf3bc34..991005c 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java
@@ -20,10 +20,8 @@ package org.apache.flink.runtime.webmonitor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService;
+import org.apache.flink.runtime.dispatcher.DispatcherGateway;
import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
import org.apache.flink.runtime.webmonitor.handlers.JarDeleteHandler;
import org.apache.flink.runtime.webmonitor.handlers.JarDeleteHeaders;
@@ -53,27 +51,15 @@ public class WebSubmissionExtension implements WebMonitorExtension {
private final ArrayList<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> webSubmissionHandlers;
- private final RestClusterClient<?> restClusterClient;
-
public WebSubmissionExtension(
Configuration configuration,
CompletableFuture<String> restAddressFuture,
- GatewayRetriever<? extends RestfulGateway> leaderRetriever,
+ GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
Map<String, String> responseHeaders,
Path jarDir,
Executor executor,
Time timeout) throws Exception {
- final SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService();
- restAddressFuture.thenAccept(restAddress -> settableLeaderRetrievalService.notifyListener(
- restAddress,
- HighAvailabilityServices.DEFAULT_LEADER_ID));
-
- restClusterClient = new RestClusterClient<>(
- configuration,
- "WebSubmissionHandlers",
- settableLeaderRetrievalService);
-
webSubmissionHandlers = new ArrayList<>(5);
final JarUploadHandler jarUploadHandler = new JarUploadHandler(
@@ -102,8 +88,7 @@ public class WebSubmissionExtension implements WebMonitorExtension {
JarRunHeaders.getInstance(),
jarDir,
configuration,
- executor,
- restClusterClient);
+ executor);
final JarDeleteHandler jarDeleteHandler = new JarDeleteHandler(
restAddressFuture,
@@ -134,8 +119,6 @@ public class WebSubmissionExtension implements WebMonitorExtension {
@Override
public CompletableFuture<Void> closeAsync() {
- restClusterClient.shutdown();
-
return CompletableFuture.completedFuture(null);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/f4bef6fc/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java
index 09b7a8b..2e928b0 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java
@@ -23,27 +23,36 @@ import org.apache.flink.api.common.time.Time;
import org.apache.flink.client.program.PackagedProgram;
import org.apache.flink.client.program.PackagedProgramUtils;
import org.apache.flink.client.program.ProgramInvocationException;
-import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.blob.BlobClient;
+import org.apache.flink.runtime.blob.PermanentBlobKey;
+import org.apache.flink.runtime.dispatcher.DispatcherGateway;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.rest.handler.AbstractRestHandler;
import org.apache.flink.runtime.rest.handler.HandlerRequest;
import org.apache.flink.runtime.rest.handler.RestHandlerException;
import org.apache.flink.runtime.rest.messages.EmptyRequestBody;
import org.apache.flink.runtime.rest.messages.MessageHeaders;
-import org.apache.flink.runtime.webmonitor.RestfulGateway;
+import org.apache.flink.runtime.util.ScalaUtils;
import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
+import org.apache.flink.util.FlinkException;
import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
+import akka.actor.AddressFromURIString;
+
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.Executor;
@@ -57,7 +66,7 @@ import static org.apache.flink.shaded.guava18.com.google.common.base.Strings.emp
* Handler to submit jobs uploaded via the Web UI.
*/
public class JarRunHandler extends
- AbstractRestHandler<RestfulGateway, EmptyRequestBody, JarRunResponseBody, JarRunMessageParameters> {
+ AbstractRestHandler<DispatcherGateway, EmptyRequestBody, JarRunResponseBody, JarRunMessageParameters> {
private final Path jarDir;
@@ -65,30 +74,26 @@ public class JarRunHandler extends
private final Executor executor;
- private final RestClusterClient<?> restClusterClient;
-
public JarRunHandler(
final CompletableFuture<String> localRestAddress,
- final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
+ final GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
final Time timeout,
final Map<String, String> responseHeaders,
final MessageHeaders<EmptyRequestBody, JarRunResponseBody, JarRunMessageParameters> messageHeaders,
final Path jarDir,
final Configuration configuration,
- final Executor executor,
- final RestClusterClient<?> restClusterClient) {
+ final Executor executor) {
super(localRestAddress, leaderRetriever, timeout, responseHeaders, messageHeaders);
this.jarDir = requireNonNull(jarDir);
this.configuration = requireNonNull(configuration);
this.executor = requireNonNull(executor);
- this.restClusterClient = requireNonNull(restClusterClient);
}
@Override
protected CompletableFuture<JarRunResponseBody> handleRequest(
@Nonnull final HandlerRequest<EmptyRequestBody, JarRunMessageParameters> request,
- @Nonnull final RestfulGateway gateway) throws RestHandlerException {
+ @Nonnull final DispatcherGateway gateway) throws RestHandlerException {
final String pathParameter = request.getPathParameter(JarIdPathParameter.class);
final Path jarFile = jarDir.resolve(pathParameter);
@@ -105,9 +110,32 @@ public class JarRunHandler extends
savepointRestoreSettings,
parallelism);
- return jobGraphFuture.thenCompose(jobGraph -> restClusterClient
- .submitJob(jobGraph)
- .thenApply((jobSubmitResponseBody -> new JarRunResponseBody(jobGraph.getJobID()))))
+ CompletableFuture<Integer> blobServerPortFuture = gateway.getBlobServerPort(timeout);
+
+ CompletableFuture<JobGraph> jarUploadFuture = jobGraphFuture.thenCombine(blobServerPortFuture, (jobGraph, blobServerPort) -> {
+ final InetSocketAddress address = new InetSocketAddress(getDispatcherHost(gateway), blobServerPort);
+ final List<PermanentBlobKey> keys;
+ try {
+ keys = BlobClient.uploadJarFiles(address, configuration, jobGraph.getJobID(), jobGraph.getUserJars());
+ } catch (IOException ioe) {
+ throw new CompletionException(new FlinkException("Could not upload job jar files.", ioe));
+ }
+
+ for (PermanentBlobKey key : keys) {
+ jobGraph.addBlob(key);
+ }
+
+ return jobGraph;
+ });
+
+ CompletableFuture<Acknowledge> jobSubmissionFuture = jarUploadFuture.thenCompose(jobGraph -> {
+ // we have to enable queued scheduling because slots will be allocated lazily
+ jobGraph.setAllowQueuedScheduling(true);
+ return gateway.submitJob(jobGraph, timeout);
+ });
+
+ return jobSubmissionFuture
+ .thenCombine(jarUploadFuture, (ack, jobGraph) -> new JarRunResponseBody(jobGraph.getJobID()))
.exceptionally(throwable -> {
throw new CompletionException(new RestHandlerException(
throwable.getMessage(),
@@ -160,4 +188,15 @@ public class JarRunHandler extends
return jobGraph;
}, executor);
}
+
+ private static String getDispatcherHost(DispatcherGateway gateway) {
+ String dispatcherAddress = gateway.getAddress();
+ final Optional<String> host = ScalaUtils.toJava(AddressFromURIString.parse(dispatcherAddress).host());
+
+ return host.orElseGet(() -> {
+ // if the dispatcher address does not contain a host part, then assume it's running
+ // on the same machine as the handler
+ return "localhost";
+ });
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/f4bef6fc/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunMessageParameters.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunMessageParameters.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunMessageParameters.java
index 2d9428c..78267db 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunMessageParameters.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunMessageParameters.java
@@ -31,17 +31,17 @@ import java.util.Collections;
*/
public class JarRunMessageParameters extends MessageParameters {
- private final JarIdPathParameter jarIdPathParameter = new JarIdPathParameter();
+ public final JarIdPathParameter jarIdPathParameter = new JarIdPathParameter();
- private final ProgramArgsQueryParameter programArgsQueryParameter = new ProgramArgsQueryParameter();
+ public final ProgramArgsQueryParameter programArgsQueryParameter = new ProgramArgsQueryParameter();
- private final EntryClassQueryParameter entryClassQueryParameter = new EntryClassQueryParameter();
+ public final EntryClassQueryParameter entryClassQueryParameter = new EntryClassQueryParameter();
- private final ParallelismQueryParameter parallelismQueryParameter = new ParallelismQueryParameter();
+ public final ParallelismQueryParameter parallelismQueryParameter = new ParallelismQueryParameter();
- private final AllowNonRestoredStateQueryParameter allowNonRestoredStateQueryParameter = new AllowNonRestoredStateQueryParameter();
+ public final AllowNonRestoredStateQueryParameter allowNonRestoredStateQueryParameter = new AllowNonRestoredStateQueryParameter();
- private final SavepointPathQueryParameter savepointPathQueryParameter = new SavepointPathQueryParameter();
+ public final SavepointPathQueryParameter savepointPathQueryParameter = new SavepointPathQueryParameter();
@Override
public Collection<MessagePathParameter<?>> getPathParameters() {
http://git-wip-us.apache.org/repos/asf/flink/blob/f4bef6fc/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java
new file mode 100644
index 0000000..aefe4f1
--- /dev/null
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.RestOptions;
+import org.apache.flink.configuration.WebOptions;
+import org.apache.flink.runtime.rest.RestClient;
+import org.apache.flink.runtime.rest.RestClientConfiguration;
+import org.apache.flink.runtime.rest.messages.EmptyRequestBody;
+import org.apache.flink.runtime.rest.util.RestClientException;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.test.util.MiniClusterResource;
+import org.apache.flink.util.ExceptionUtils;
+
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for the {@link JarRunHandler}.
+ */
+public class JarRunHandlerTest {
+
+ @ClassRule
+ public static final TemporaryFolder TMP = new TemporaryFolder();
+
+ @Test
+ public void testRunJar() throws Exception {
+ Path uploadDir = TMP.newFolder().toPath();
+
+ Path actualUploadDir = uploadDir.resolve("flink-web-upload");
+ Files.createDirectory(actualUploadDir);
+
+ Path emptyJar = actualUploadDir.resolve("empty.jar");
+ Files.createFile(emptyJar);
+
+ Configuration config = new Configuration();
+ config.setString(WebOptions.UPLOAD_DIR, uploadDir.toString());
+
+ MiniClusterResource clusterResource = new MiniClusterResource(
+ new MiniClusterResource.MiniClusterResourceConfiguration(
+ config,
+ 1,
+ 1
+ ),
+ MiniClusterResource.MiniClusterType.NEW
+ );
+ clusterResource.before();
+
+ try {
+ Configuration clientConfig = clusterResource.getClientConfiguration();
+ RestClient client = new RestClient(RestClientConfiguration.fromConfiguration(clientConfig), TestingUtils.defaultExecutor());
+
+ try {
+ JarRunHeaders headers = JarRunHeaders.getInstance();
+ JarRunMessageParameters parameters = headers.getUnresolvedMessageParameters();
+ parameters.jarIdPathParameter.resolve(emptyJar.getFileName().toString());
+
+ String host = clientConfig.getString(RestOptions.ADDRESS);
+ int port = clientConfig.getInteger(RestOptions.PORT);
+
+ try {
+ client.sendRequest(host, port, headers, parameters, EmptyRequestBody.getInstance())
+ .get();
+ } catch (Exception e) {
+ Optional<RestClientException> expected = ExceptionUtils.findThrowable(e, RestClientException.class);
+ if (expected.isPresent()) {
+ // implies the job was actually submitted
+ assertTrue(expected.get().getMessage().contains("ProgramInvocationException"));
+ // implies the jar was registered for the job graph (otherwise the jar name would not occur in the exception)
+ // implies the jar was uploaded (otherwise the file would not be found at all)
+ assertTrue(expected.get().getMessage().contains("empty.jar'. zip file is empty"));
+ } else {
+ throw e;
+ }
+ }
+ } finally {
+ client.shutdown(Time.milliseconds(10));
+ }
+ } finally {
+ clusterResource.after();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f4bef6fc/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
index 24ecf0c..4b27534 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
@@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.WebOptions;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.concurrent.ScheduledExecutor;
+import org.apache.flink.runtime.dispatcher.DispatcherGateway;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
import org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex;
@@ -219,7 +220,7 @@ public final class WebMonitorUtils {
* @throws FlinkException if the web submission extension could not be loaded
*/
public static WebMonitorExtension loadWebSubmissionExtension(
- GatewayRetriever<? extends RestfulGateway> leaderRetriever,
+ GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
CompletableFuture<String> restAddressFuture,
Time timeout,
Map<String, String> responseHeaders,
[7/7] flink git commit: [hotfix][docs] Fix typehint in state
documentation
Posted by ch...@apache.org.
[hotfix][docs] Fix typehint in state documentation
This closes #5919.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dd563916
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dd563916
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dd563916
Branch: refs/heads/release-1.5
Commit: dd5639164bf72cab7f54b7f82d50b68d6d2c7a7c
Parents: 1c48e32
Author: maqingxiang-it <ma...@dev05v.sys.corp.qihoo.net>
Authored: Thu Apr 26 18:14:07 2018 +0800
Committer: zentol <ch...@apache.org>
Committed: Wed May 2 15:19:33 2018 +0200
----------------------------------------------------------------------
docs/dev/stream/state/state.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/dd563916/docs/dev/stream/state/state.md
----------------------------------------------------------------------
diff --git a/docs/dev/stream/state/state.md b/docs/dev/stream/state/state.md
index a26e24d..44a3653 100644
--- a/docs/dev/stream/state/state.md
+++ b/docs/dev/stream/state/state.md
@@ -368,7 +368,7 @@ public class BufferingSink
ListStateDescriptor<Tuple2<String, Integer>> descriptor =
new ListStateDescriptor<>(
"buffered-elements",
- TypeInformation.of(new TypeHint<Tuple2<Long, Long>>() {}));
+ TypeInformation.of(new TypeHint<Tuple2<String, Integer>>() {}));
checkpointedState = context.getOperatorStateStore().getListState(descriptor);
[6/7] flink git commit: [FLINK-9154][REST][docs] Document
WebSubmissionExtension handlers
Posted by ch...@apache.org.
[FLINK-9154][REST][docs] Document WebSubmissionExtension handlers
This closes #5833.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1c48e324
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1c48e324
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1c48e324
Branch: refs/heads/release-1.5
Commit: 1c48e32446363e903f5084082e9b9821c0db4587
Parents: f4bef6f
Author: zentol <ch...@apache.org>
Authored: Tue Apr 10 13:38:10 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed May 2 15:19:32 2018 +0200
----------------------------------------------------------------------
docs/_includes/generated/rest_dispatcher.html | 307 +++++++++++++++++++
flink-docs/pom.xml | 6 +
.../flink/docs/rest/RestAPIDocGenerator.java | 3 +
3 files changed, 316 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1c48e324/docs/_includes/generated/rest_dispatcher.html
----------------------------------------------------------------------
diff --git a/docs/_includes/generated/rest_dispatcher.html b/docs/_includes/generated/rest_dispatcher.html
index 98c9d42..36bf242 100644
--- a/docs/_includes/generated/rest_dispatcher.html
+++ b/docs/_includes/generated/rest_dispatcher.html
@@ -101,6 +101,313 @@
<table class="table table-bordered">
<tbody>
<tr>
+ <td class="text-left" colspan="2"><strong>/jars</strong></td>
+ </tr>
+ <tr>
+ <td class="text-left" style="width: 20%">Verb: <code>GET</code></td>
+ <td class="text-left">Response code: <code>200 OK</code></td>
+ </tr>
+ <tr>
+ <td colspan="2">description</td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#-525177469">Request</button>
+ <div id="-525177469" class="collapse">
+ <pre>
+ <code>
+{} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#-1669843955">Response</button>
+ <div id="-1669843955" class="collapse">
+ <pre>
+ <code>
+{
+ "type" : "object",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarListInfo",
+ "properties" : {
+ "address" : {
+ "type" : "string"
+ },
+ "files" : {
+ "type" : "array",
+ "items" : {
+ "type" : "object",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarListInfo:JarFileInfo",
+ "properties" : {
+ "id" : {
+ "type" : "string"
+ },
+ "name" : {
+ "type" : "string"
+ },
+ "uploaded" : {
+ "type" : "integer"
+ },
+ "entry" : {
+ "type" : "array",
+ "items" : {
+ "type" : "object",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarListInfo:JarEntryInfo",
+ "properties" : {
+ "name" : {
+ "type" : "string"
+ },
+ "description" : {
+ "type" : "string"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+</table>
+<table class="table table-bordered">
+ <tbody>
+ <tr>
+ <td class="text-left" colspan="2"><strong>/jars/upload</strong></td>
+ </tr>
+ <tr>
+ <td class="text-left" style="width: 20%">Verb: <code>POST</code></td>
+ <td class="text-left">Response code: <code>200 OK</code></td>
+ </tr>
+ <tr>
+ <td colspan="2">description</td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#919877128">Request</button>
+ <div id="919877128" class="collapse">
+ <pre>
+ <code>
+{
+ "type" : "object",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:FileUpload",
+ "properties" : {
+ "path" : {
+ "type" : "string"
+ }
+ }
+} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#1132420209">Response</button>
+ <div id="1132420209" class="collapse">
+ <pre>
+ <code>
+{
+ "type" : "object",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarUploadResponseBody",
+ "properties" : {
+ "filename" : {
+ "type" : "string"
+ },
+ "status" : {
+ "type" : "string",
+ "enum" : [ "success" ]
+ }
+ }
+} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+</table>
+<table class="table table-bordered">
+ <tbody>
+ <tr>
+ <td class="text-left" colspan="2"><strong>/jars/:jarid</strong></td>
+ </tr>
+ <tr>
+ <td class="text-left" style="width: 20%">Verb: <code>DELETE</code></td>
+ <td class="text-left">Response code: <code>200 OK</code></td>
+ </tr>
+ <tr>
+ <td colspan="2">description</td>
+ </tr>
+ <tr>
+ <td colspan="2">Path parameters</td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <ul>
+<li><code>jarid</code> - description</li>
+ </ul>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#-974639435">Request</button>
+ <div id="-974639435" class="collapse">
+ <pre>
+ <code>
+{} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#-789656611">Response</button>
+ <div id="-789656611" class="collapse">
+ <pre>
+ <code>
+{} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+</table>
+<table class="table table-bordered">
+ <tbody>
+ <tr>
+ <td class="text-left" colspan="2"><strong>/jars/:jarid/plan</strong></td>
+ </tr>
+ <tr>
+ <td class="text-left" style="width: 20%">Verb: <code>GET</code></td>
+ <td class="text-left">Response code: <code>200 OK</code></td>
+ </tr>
+ <tr>
+ <td colspan="2">description</td>
+ </tr>
+ <tr>
+ <td colspan="2">Path parameters</td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <ul>
+<li><code>jarid</code> - description</li>
+ </ul>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">Query parameters</td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <ul>
+<li><code>entry-class</code> (optional): description</li>
+<li><code>parallelism</code> (optional): description</li>
+<li><code>program-args</code> (optional): description</li>
+ </ul>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#-181694384">Request</button>
+ <div id="-181694384" class="collapse">
+ <pre>
+ <code>
+{} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#1311035100">Response</button>
+ <div id="1311035100" class="collapse">
+ <pre>
+ <code>
+{
+ "type" : "any"
+} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+</table>
+<table class="table table-bordered">
+ <tbody>
+ <tr>
+ <td class="text-left" colspan="2"><strong>/jars/:jarid/run</strong></td>
+ </tr>
+ <tr>
+ <td class="text-left" style="width: 20%">Verb: <code>POST</code></td>
+ <td class="text-left">Response code: <code>200 OK</code></td>
+ </tr>
+ <tr>
+ <td colspan="2">description</td>
+ </tr>
+ <tr>
+ <td colspan="2">Path parameters</td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <ul>
+<li><code>jarid</code> - description</li>
+ </ul>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">Query parameters</td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <ul>
+<li><code>program-args</code> (optional): description</li>
+<li><code>entry-class</code> (optional): description</li>
+<li><code>parallelism</code> (optional): description</li>
+<li><code>allowNonRestoredState</code> (optional): description</li>
+<li><code>savepointPath</code> (optional): description</li>
+ </ul>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#1936993190">Request</button>
+ <div id="1936993190" class="collapse">
+ <pre>
+ <code>
+{} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <button data-toggle="collapse" data-target="#-914083094">Response</button>
+ <div id="-914083094" class="collapse">
+ <pre>
+ <code>
+{
+ "type" : "object",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:webmonitor:handlers:JarRunResponseBody",
+ "properties" : {
+ "jobid" : {
+ "type" : "any"
+ }
+ }
+} </code>
+ </pre>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+</table>
+<table class="table table-bordered">
+ <tbody>
+ <tr>
<td class="text-left" colspan="2"><strong>/jobmanager/config</strong></td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/flink/blob/1c48e324/flink-docs/pom.xml
----------------------------------------------------------------------
diff --git a/flink-docs/pom.xml b/flink-docs/pom.xml
index c8747a5..903bbce 100644
--- a/flink-docs/pom.xml
+++ b/flink-docs/pom.xml
@@ -54,6 +54,12 @@ under the License.
<version>${project.version}</version>
</dependency>
<dependency>
+ <!-- necessary for loading the web-submission extension -->
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-yarn_${scala.binary.version}</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/1c48e324/flink-docs/src/main/java/org/apache/flink/docs/rest/RestAPIDocGenerator.java
----------------------------------------------------------------------
diff --git a/flink-docs/src/main/java/org/apache/flink/docs/rest/RestAPIDocGenerator.java b/flink-docs/src/main/java/org/apache/flink/docs/rest/RestAPIDocGenerator.java
index 79bf677..70734da 100644
--- a/flink-docs/src/main/java/org/apache/flink/docs/rest/RestAPIDocGenerator.java
+++ b/flink-docs/src/main/java/org/apache/flink/docs/rest/RestAPIDocGenerator.java
@@ -20,6 +20,7 @@ package org.apache.flink.docs.rest;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.configuration.RestOptions;
import org.apache.flink.runtime.concurrent.Executors;
import org.apache.flink.runtime.dispatcher.DispatcherGateway;
@@ -314,6 +315,8 @@ public class RestAPIDocGenerator {
static {
config = new Configuration();
config.setString(RestOptions.ADDRESS, "localhost");
+ // necessary for loading the web-submission extension
+ config.setString(JobManagerOptions.ADDRESS, "localhost");
try {
restConfig = RestServerEndpointConfiguration.fromConfiguration(config);
} catch (ConfigurationException e) {
[5/7] flink git commit: [FLINK-8743][docs] Allow overriding
documented default
Posted by ch...@apache.org.
[FLINK-8743][docs] Allow overriding documented default
This closes #5822.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d83e19fb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d83e19fb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d83e19fb
Branch: refs/heads/release-1.5
Commit: d83e19fbb20e4e04b6526079db7f4d00d687fae6
Parents: 4535489
Author: zentol <ch...@apache.org>
Authored: Thu Apr 5 12:42:42 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed May 2 15:19:31 2018 +0200
----------------------------------------------------------------------
.../_includes/generated/core_configuration.html | 2 +-
docs/_includes/generated/web_configuration.html | 2 +-
.../flink/annotation/docs/Documentation.java | 45 +++++++++++++
.../apache/flink/configuration/CoreOptions.java | 2 +
.../apache/flink/configuration/WebOptions.java | 2 +
.../ConfigOptionsDocGenerator.java | 69 ++++++++++++--------
.../ConfigOptionsDocGeneratorTest.java | 44 +++++++++++++
7 files changed, 135 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d83e19fb/docs/_includes/generated/core_configuration.html
----------------------------------------------------------------------
diff --git a/docs/_includes/generated/core_configuration.html b/docs/_includes/generated/core_configuration.html
index 606fd5b..5e29cb9 100644
--- a/docs/_includes/generated/core_configuration.html
+++ b/docs/_includes/generated/core_configuration.html
@@ -24,7 +24,7 @@
</tr>
<tr>
<td><h5>io.tmp.dirs</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
+ <td style="word-wrap: break-word;">System.getProperty("java.io.tmpdir")</td>
<td></td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/flink/blob/d83e19fb/docs/_includes/generated/web_configuration.html
----------------------------------------------------------------------
diff --git a/docs/_includes/generated/web_configuration.html b/docs/_includes/generated/web_configuration.html
index 23bf5ab..d4f5a7e 100644
--- a/docs/_includes/generated/web_configuration.html
+++ b/docs/_includes/generated/web_configuration.html
@@ -79,7 +79,7 @@
</tr>
<tr>
<td><h5>web.tmpdir</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
+ <td style="word-wrap: break-word;">System.getProperty("java.io.tmpdir")</td>
<td></td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/flink/blob/d83e19fb/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
----------------------------------------------------------------------
diff --git a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
new file mode 100644
index 0000000..4e424c7
--- /dev/null
+++ b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.annotation.docs;
+
+import org.apache.flink.annotation.Internal;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Collection of annotations to modify the behavior of the documentation generators.
+ */
+public final class Documentation {
+
+ /**
+ * Annotation used on config option fields to override the documented default.
+ */
+ @Target(ElementType.FIELD)
+ @Retention(RetentionPolicy.RUNTIME)
+ @Internal
+ public @interface OverrideDefault {
+ String value();
+ }
+
+ private Documentation(){
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/d83e19fb/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
index 343e2d2..42c1c9c 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
@@ -21,6 +21,7 @@ package org.apache.flink.configuration;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.docs.ConfigGroup;
import org.apache.flink.annotation.docs.ConfigGroups;
+import org.apache.flink.annotation.docs.Documentation;
import static org.apache.flink.configuration.ConfigOptions.key;
@@ -180,6 +181,7 @@ public class CoreOptions {
* The config parameter defining the directories for temporary files, separated by
* ",", "|", or the system's {@link java.io.File#pathSeparator}.
*/
+ @Documentation.OverrideDefault("System.getProperty(\"java.io.tmpdir\")")
public static final ConfigOption<String> TMP_DIRS =
key("io.tmp.dirs")
.defaultValue(System.getProperty("java.io.tmpdir"))
http://git-wip-us.apache.org/repos/asf/flink/blob/d83e19fb/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java
index 7eea17d..ff640cc 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java
@@ -19,6 +19,7 @@
package org.apache.flink.configuration;
import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.annotation.docs.Documentation;
import static org.apache.flink.configuration.ConfigOptions.key;
@@ -72,6 +73,7 @@ public class WebOptions {
/**
* The config parameter defining the flink web directory to be used by the webmonitor.
*/
+ @Documentation.OverrideDefault("System.getProperty(\"java.io.tmpdir\")")
public static final ConfigOption<String> TMP_DIR =
key("web.tmpdir")
.defaultValue(System.getProperty("java.io.tmpdir"))
http://git-wip-us.apache.org/repos/asf/flink/blob/d83e19fb/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
----------------------------------------------------------------------
diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
index b51a028..aadeee6 100644
--- a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
+++ b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
@@ -21,10 +21,9 @@ package org.apache.flink.docs.configuration;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.annotation.docs.ConfigGroup;
import org.apache.flink.annotation.docs.ConfigGroups;
+import org.apache.flink.annotation.docs.Documentation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.CoreOptions;
-import org.apache.flink.configuration.WebOptions;
import org.apache.flink.util.function.ThrowingConsumer;
import java.io.IOException;
@@ -126,7 +125,7 @@ public class ConfigOptionsDocGenerator {
@VisibleForTesting
static List<Tuple2<ConfigGroup, String>> generateTablesForClass(Class<?> optionsClass) {
ConfigGroups configGroups = optionsClass.getAnnotation(ConfigGroups.class);
- List<ConfigOption<?>> allOptions = extractConfigOptions(optionsClass);
+ List<OptionWithMetaInfo> allOptions = extractConfigOptions(optionsClass);
List<Tuple2<ConfigGroup, String>> tables;
if (configGroups != null) {
@@ -134,11 +133,11 @@ public class ConfigOptionsDocGenerator {
Tree tree = new Tree(configGroups.groups(), allOptions);
for (ConfigGroup group : configGroups.groups()) {
- List<ConfigOption<?>> configOptions = tree.findConfigOptions(group);
+ List<OptionWithMetaInfo> configOptions = tree.findConfigOptions(group);
sortOptions(configOptions);
tables.add(Tuple2.of(group, toHtmlTable(configOptions)));
}
- List<ConfigOption<?>> configOptions = tree.getDefaultOptions();
+ List<OptionWithMetaInfo> configOptions = tree.getDefaultOptions();
sortOptions(configOptions);
tables.add(Tuple2.of(null, toHtmlTable(configOptions)));
} else {
@@ -148,13 +147,13 @@ public class ConfigOptionsDocGenerator {
return tables;
}
- private static List<ConfigOption<?>> extractConfigOptions(Class<?> clazz) {
+ private static List<OptionWithMetaInfo> extractConfigOptions(Class<?> clazz) {
try {
- List<ConfigOption<?>> configOptions = new ArrayList<>(8);
+ List<OptionWithMetaInfo> configOptions = new ArrayList<>(8);
Field[] fields = clazz.getFields();
for (Field field : fields) {
if (field.getType().equals(ConfigOption.class) && field.getAnnotation(Deprecated.class) == null) {
- configOptions.add((ConfigOption<?>) field.get(null));
+ configOptions.add(new OptionWithMetaInfo((ConfigOption<?>) field.get(null), field));
}
}
@@ -171,7 +170,7 @@ public class ConfigOptionsDocGenerator {
* @param options list of options to include in this group
* @return string containing HTML formatted table
*/
- private static String toHtmlTable(final List<ConfigOption<?>> options) {
+ private static String toHtmlTable(final List<OptionWithMetaInfo> options) {
StringBuilder htmlTable = new StringBuilder();
htmlTable.append("<table class=\"table table-bordered\">\n");
htmlTable.append(" <thead>\n");
@@ -183,7 +182,7 @@ public class ConfigOptionsDocGenerator {
htmlTable.append(" </thead>\n");
htmlTable.append(" <tbody>\n");
- for (ConfigOption<?> option : options) {
+ for (OptionWithMetaInfo option : options) {
htmlTable.append(toHtmlString(option));
}
@@ -196,21 +195,23 @@ public class ConfigOptionsDocGenerator {
/**
* Transforms option to table row.
*
- * @param option option to transform
+ * @param optionWithMetaInfo option to transform
* @return row with the option description
*/
- private static String toHtmlString(final ConfigOption<?> option) {
- Object defaultValue = option.defaultValue();
- // This is a temporary hack that should be removed once FLINK-6490 is resolved.
- // These options use System.getProperty("java.io.tmpdir") as the default.
- // As a result the generated table contains an actual path as the default, which is simply wrong.
- if (option == WebOptions.TMP_DIR || option.key().equals("python.dc.tmp.dir") || option == CoreOptions.TMP_DIRS) {
- defaultValue = null;
+ private static String toHtmlString(final OptionWithMetaInfo optionWithMetaInfo) {
+ ConfigOption<?> option = optionWithMetaInfo.option;
+ String defaultValue;
+
+ Documentation.OverrideDefault overrideDocumentedDefault = optionWithMetaInfo.field.getAnnotation(Documentation.OverrideDefault.class);
+ if (overrideDocumentedDefault != null) {
+ defaultValue = escapeCharacters(addWordBreakOpportunities(overrideDocumentedDefault.value()));
+ } else {
+ defaultValue = escapeCharacters(addWordBreakOpportunities(defaultValueToHtml(option.defaultValue())));
}
return "" +
" <tr>\n" +
" <td><h5>" + escapeCharacters(option.key()) + "</h5></td>\n" +
- " <td style=\"word-wrap: break-word;\">" + escapeCharacters(addWordBreakOpportunities(defaultValueToHtml(defaultValue))) + "</td>\n" +
+ " <td style=\"word-wrap: break-word;\">" + defaultValue + "</td>\n" +
" <td>" + escapeCharacters(option.description()) + "</td>\n" +
" </tr>\n";
}
@@ -240,8 +241,8 @@ public class ConfigOptionsDocGenerator {
.replace(";", ";<wbr>");
}
- private static void sortOptions(List<ConfigOption<?>> configOptions) {
- configOptions.sort(Comparator.comparing(ConfigOption::key));
+ private static void sortOptions(List<OptionWithMetaInfo> configOptions) {
+ configOptions.sort(Comparator.comparing(option -> option.option.key()));
}
/**
@@ -251,7 +252,7 @@ public class ConfigOptionsDocGenerator {
private static class Tree {
private final Node root = new Node();
- Tree(ConfigGroup[] groups, Collection<ConfigOption<?>> options) {
+ Tree(ConfigGroup[] groups, Collection<OptionWithMetaInfo> options) {
// generate a tree based on all key prefixes
for (ConfigGroup group : groups) {
String[] keyComponents = group.keyPrefix().split("\\.");
@@ -264,17 +265,17 @@ public class ConfigOptionsDocGenerator {
// assign options to their corresponding group, i.e. the last group root node encountered when traversing
// the tree based on the option key
- for (ConfigOption<?> option : options) {
- findGroupRoot(option.key()).assignOption(option);
+ for (OptionWithMetaInfo option : options) {
+ findGroupRoot(option.option.key()).assignOption(option);
}
}
- List<ConfigOption<?>> findConfigOptions(ConfigGroup configGroup) {
+ List<OptionWithMetaInfo> findConfigOptions(ConfigGroup configGroup) {
Node groupRoot = findGroupRoot(configGroup.keyPrefix());
return groupRoot.getConfigOptions();
}
- List<ConfigOption<?>> getDefaultOptions() {
+ List<OptionWithMetaInfo> getDefaultOptions() {
return root.getConfigOptions();
}
@@ -288,7 +289,7 @@ public class ConfigOptionsDocGenerator {
}
private static class Node {
- private final List<ConfigOption<?>> configOptions = new ArrayList<>(8);
+ private final List<OptionWithMetaInfo> configOptions = new ArrayList<>(8);
private final Map<String, Node> children = new HashMap<>(8);
private boolean isGroupRoot = false;
@@ -309,7 +310,7 @@ public class ConfigOptionsDocGenerator {
return child;
}
- private void assignOption(ConfigOption<?> option) {
+ private void assignOption(OptionWithMetaInfo option) {
configOptions.add(option);
}
@@ -321,12 +322,22 @@ public class ConfigOptionsDocGenerator {
this.isGroupRoot = true;
}
- private List<ConfigOption<?>> getConfigOptions() {
+ private List<OptionWithMetaInfo> getConfigOptions() {
return configOptions;
}
}
}
+ private static class OptionWithMetaInfo {
+ private final ConfigOption<?> option;
+ private final Field field;
+
+ public OptionWithMetaInfo(ConfigOption<?> option, Field field) {
+ this.option = option;
+ this.field = field;
+ }
+ }
+
private ConfigOptionsDocGenerator() {
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d83e19fb/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
----------------------------------------------------------------------
diff --git a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
index ba30617..4176c5d 100644
--- a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
+++ b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
@@ -20,6 +20,7 @@ package org.apache.flink.docs.configuration;
import org.apache.flink.annotation.docs.ConfigGroup;
import org.apache.flink.annotation.docs.ConfigGroups;
+import org.apache.flink.annotation.docs.Documentation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
@@ -171,4 +172,47 @@ public class ConfigOptionsDocGeneratorTest {
"</table>\n", tablesConverted.get("default"));
}
+ static class TestConfigGroupWithOverriddenDefault {
+ @Documentation.OverrideDefault("default_1")
+ public static ConfigOption<Integer> firstOption = ConfigOptions
+ .key("first.option.a")
+ .defaultValue(2)
+ .withDescription("This is example description for the first option.");
+
+ @Documentation.OverrideDefault("default_2")
+ public static ConfigOption<String> secondOption = ConfigOptions
+ .key("second.option.a")
+ .noDefaultValue()
+ .withDescription("This is long example description for the second option.");
+ }
+
+ @Test
+ public void testOverrideDefault() {
+ final String expectedTable =
+ "<table class=\"table table-bordered\">\n" +
+ " <thead>\n" +
+ " <tr>\n" +
+ " <th class=\"text-left\" style=\"width: 20%\">Key</th>\n" +
+ " <th class=\"text-left\" style=\"width: 15%\">Default</th>\n" +
+ " <th class=\"text-left\" style=\"width: 65%\">Description</th>\n" +
+ " </tr>\n" +
+ " </thead>\n" +
+ " <tbody>\n" +
+ " <tr>\n" +
+ " <td><h5>first.option.a</h5></td>\n" +
+ " <td style=\"word-wrap: break-word;\">default_1</td>\n" +
+ " <td>This is example description for the first option.</td>\n" +
+ " </tr>\n" +
+ " <tr>\n" +
+ " <td><h5>second.option.a</h5></td>\n" +
+ " <td style=\"word-wrap: break-word;\">default_2</td>\n" +
+ " <td>This is long example description for the second option.</td>\n" +
+ " </tr>\n" +
+ " </tbody>\n" +
+ "</table>\n";
+ final String htmlTable = ConfigOptionsDocGenerator.generateTablesForClass(TestConfigGroupWithOverriddenDefault.class).get(0).f1;
+
+ assertEquals(expectedTable, htmlTable);
+ }
+
}