You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/07/13 07:18:29 UTC
[1/4] flink git commit: [FLINK-9091][build] Dependency convergence
run against dependency-reduced poms
Repository: flink
Updated Branches:
refs/heads/release-1.5 8a564f82a -> 2685d0b78
[FLINK-9091][build] Dependency convergence run against dependency-reduced poms
This closes #6102.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2685d0b7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2685d0b7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2685d0b7
Branch: refs/heads/release-1.5
Commit: 2685d0b78f3475f0b46e6347f2e4e8a727ce26d9
Parents: aa160b5
Author: zentol <ch...@apache.org>
Authored: Mon May 28 15:10:53 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Fri Jul 13 06:10:32 2018 +0200
----------------------------------------------------------------------
pom.xml | 25 +++++++++++
tools/check_dependency_convergence.sh | 67 ++++++++++++++++++++++++++++++
tools/travis_mvn_watchdog.sh | 36 ++++++++++++++--
3 files changed, 124 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2685d0b7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index aab798b..79ba41f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -694,6 +694,29 @@ under the License.
</profile>
<profile>
+ <id>check-convergence</id>
+ <activation>
+ <property>
+ <name>check-convergence</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dependency-convergence</id>
+ <phase>validate</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <profile>
<id>legacyCode</id>
<activation>
<property>
@@ -1269,6 +1292,8 @@ under the License.
</execution>
<execution>
<id>dependency-convergence</id>
+ <!-- disabled by default as it interacts badly with shade-plugin -->
+ <phase>none</phase>
<goals>
<goal>enforce</goal>
</goals>
http://git-wip-us.apache.org/repos/asf/flink/blob/2685d0b7/tools/check_dependency_convergence.sh
----------------------------------------------------------------------
diff --git a/tools/check_dependency_convergence.sh b/tools/check_dependency_convergence.sh
new file mode 100755
index 0000000..5db8d9d
--- /dev/null
+++ b/tools/check_dependency_convergence.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+HERE="`dirname \"$0\"`" # relative
+HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized
+if [ -z "$HERE" ] ; then
+ # error; for some reason, the path is not accessible
+ # to the script (e.g. permissions re-evaled after suid)
+ exit 1 # fail
+fi
+
+FLINK_DIR=HERE
+
+if [[ $(basename ${HERE}) == "tools" ]] ; then
+ FLINK_DIR="${HERE}/.."
+fi
+
+FLINK_DIR="`( cd \"${FLINK_DIR}\" && pwd )`"
+
+echo ${FLINK_DIR}
+
+# get list of all flink modules
+# searches for directories containing a pom.xml file
+# sorts the list alphabetically
+# only accepts directories starting with "flink" to filter force-shading
+modules=$(find . -maxdepth 3 -name 'pom.xml' -printf '%h\n' | sort -u | grep "flink")
+
+for module in ${modules}
+do
+ # we are only interested in child modules
+ for other_module in ${modules}
+ do
+ if [[ "${other_module}" != "${module}" && "${other_module}" = "${module}"/* ]]; then
+ echo "excluding ${module} since it is not a leaf module"
+ continue 2
+ fi
+ done
+
+ cd "${module}"
+ echo "checking ${module}"
+ output=$(mvn validate -nsu -Dcheckstyle.skip=true -Dcheck-convergence)
+ exit_code=$?
+ if [[ ${exit_code} != 0 ]]; then
+ echo "dependency convergence failed."
+ echo "${output}"
+ exit ${exit_code}
+ fi
+ cd "${FLINK_DIR}"
+done
+
+exit 0
http://git-wip-us.apache.org/repos/asf/flink/blob/2685d0b7/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index e08f0cc..d008124 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -500,12 +500,40 @@ EXIT_CODE=$(<$MVN_EXIT)
echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
+# Make sure to kill the watchdog in any case after $MVN_COMPILE has completed
+echo "Trying to KILL watchdog (${WD_PID})."
+( kill $WD_PID 2>&1 ) > /dev/null
+
rm $MVN_PID
rm $MVN_EXIT
+# only run dependency-convergence in misc because it is the only profile building all of Flink
+case $TEST in
+ (misc)
+ if [ $EXIT_CODE == 0 ]; then
+ printf "\n\n==============================================================================\n"
+ printf "Checking dependency convergence\n"
+ printf "==============================================================================\n"
+
+ ./tools/check_dependency_convergence.sh
+ EXIT_CODE=$?
+ else
+ printf "\n==============================================================================\n"
+ printf "Previous build failure detected, skipping dependency-convergence check.\n"
+ printf "==============================================================================\n"
+ fi
+ ;;
+esac
+
# Run tests if compilation was successful
if [ $EXIT_CODE == 0 ]; then
+ # Start watching $MVN_OUT
+ watchdog &
+ echo "STARTED watchdog (${WD_PID})."
+
+ WD_PID=$!
+
echo "RUNNING '${MVN_TEST}'."
# Run $MVN_TEST and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
@@ -517,6 +545,10 @@ if [ $EXIT_CODE == 0 ]; then
echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
+ # Make sure to kill the watchdog in any case after $MVN_TEST has completed
+ echo "Trying to KILL watchdog (${WD_PID})."
+ ( kill $WD_PID 2>&1 ) > /dev/null
+
rm $MVN_PID
rm $MVN_EXIT
else
@@ -527,10 +559,6 @@ fi
# Post
-# Make sure to kill the watchdog in any case after $MVN_COMPILE and $MVN_TEST have completed
-echo "Trying to KILL watchdog (${WD_PID})."
-( kill $WD_PID 2>&1 ) > /dev/null
-
# only misc builds flink-dist and flink-yarn-tests
case $TEST in
(misc)
[3/4] flink git commit: [FLINK-9771][rest] Fix plan JSON response
Posted by ch...@apache.org.
[FLINK-9771][rest] Fix plan JSON response
This closes #6274.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/aa160b52
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/aa160b52
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/aa160b52
Branch: refs/heads/release-1.5
Commit: aa160b528922c1e0781941b3d2140540919ac326
Parents: 2f7ab90
Author: zentol <ch...@apache.org>
Authored: Fri Jul 6 11:22:20 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Fri Jul 13 06:10:32 2018 +0200
----------------------------------------------------------------------
.../runtime/rest/messages/JobPlanInfo.java | 115 +++++++++++++------
.../runtime/rest/messages/JobPlanInfoTest.java | 9 +-
2 files changed, 86 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/aa160b52/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobPlanInfo.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobPlanInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobPlanInfo.java
index 11cb89b..7263b36 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobPlanInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobPlanInfo.java
@@ -21,9 +21,12 @@ package org.apache.flink.runtime.rest.messages;
import org.apache.flink.runtime.rest.handler.job.JobPlanHandler;
import org.apache.flink.util.Preconditions;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParser;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationContext;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.SerializerProvider;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.annotation.JsonSerialize;
@@ -36,18 +39,20 @@ import java.util.Objects;
/**
* Response type of the {@link JobPlanHandler}.
*/
-@JsonSerialize(using = JobPlanInfo.Serializer.class)
-@JsonDeserialize(using = JobPlanInfo.Deserializer.class)
public class JobPlanInfo implements ResponseBody {
- private final String jsonPlan;
+ private static final String FIELD_NAME_PLAN = "plan";
+
+ @JsonProperty(FIELD_NAME_PLAN)
+ private final RawJson jsonPlan;
public JobPlanInfo(String jsonPlan) {
- this.jsonPlan = Preconditions.checkNotNull(jsonPlan);
+ this(new RawJson(Preconditions.checkNotNull(jsonPlan)));
}
- public String getJsonPlan() {
- return jsonPlan;
+ @JsonCreator
+ public JobPlanInfo(@JsonProperty(FIELD_NAME_PLAN) RawJson jsonPlan) {
+ this.jsonPlan = jsonPlan;
}
@Override
@@ -67,47 +72,91 @@ public class JobPlanInfo implements ResponseBody {
return Objects.hash(jsonPlan);
}
- //---------------------------------------------------------------------------------
- // Static helper classes
- //---------------------------------------------------------------------------------
+ @Override
+ public String toString() {
+ return "JobPlanInfo{" +
+ "jsonPlan=" + jsonPlan +
+ '}';
+ }
/**
- * Json serializer for the {@link JobPlanInfo}.
+ * Simple wrapper around a raw JSON string.
*/
- public static final class Serializer extends StdSerializer<JobPlanInfo> {
+ @JsonSerialize(using = RawJson.Serializer.class)
+ @JsonDeserialize(using = RawJson.Deserializer.class)
+ public static final class RawJson {
+ private final String json;
- private static final long serialVersionUID = -1551666039618928811L;
+ private RawJson(String json) {
+ this.json = json;
+ }
- public Serializer() {
- super(JobPlanInfo.class);
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ RawJson rawJson = (RawJson) o;
+ return Objects.equals(json, rawJson.json);
}
@Override
- public void serialize(
- JobPlanInfo jobPlanInfo,
- JsonGenerator jsonGenerator,
- SerializerProvider serializerProvider) throws IOException {
- jsonGenerator.writeString(jobPlanInfo.getJsonPlan());
+ public int hashCode() {
+ return Objects.hash(json);
}
- }
- /**
- * Json deserializer for the {@link JobPlanInfo}.
- */
- public static final class Deserializer extends StdDeserializer<JobPlanInfo> {
+ @Override
+ public String toString() {
+ return "RawJson{" +
+ "json='" + json + '\'' +
+ '}';
+ }
+
+ //---------------------------------------------------------------------------------
+ // Static helper classes
+ //---------------------------------------------------------------------------------
+
+ /**
+ * Json serializer for the {@link RawJson}.
+ */
+ public static final class Serializer extends StdSerializer<RawJson> {
+
+ private static final long serialVersionUID = -1551666039618928811L;
- private static final long serialVersionUID = -3580088509877177213L;
+ public Serializer() {
+ super(RawJson.class);
+ }
- public Deserializer() {
- super(JobPlanInfo.class);
+ @Override
+ public void serialize(
+ RawJson jobPlanInfo,
+ JsonGenerator jsonGenerator,
+ SerializerProvider serializerProvider) throws IOException {
+ jsonGenerator.writeRawValue(jobPlanInfo.json);
+ }
}
- @Override
- public JobPlanInfo deserialize(
- JsonParser jsonParser,
- DeserializationContext deserializationContext) throws IOException {
- final String jsonPlan = jsonParser.getText();
- return new JobPlanInfo(jsonPlan);
+ /**
+ * Json deserializer for the {@link RawJson}.
+ */
+ public static final class Deserializer extends StdDeserializer<RawJson> {
+
+ private static final long serialVersionUID = -3580088509877177213L;
+
+ public Deserializer() {
+ super(RawJson.class);
+ }
+
+ @Override
+ public RawJson deserialize(
+ JsonParser jsonParser,
+ DeserializationContext deserializationContext) throws IOException {
+ final JsonNode rootNode = jsonParser.readValueAsTree();
+ return new RawJson(rootNode.toString());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/aa160b52/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobPlanInfoTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobPlanInfoTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobPlanInfoTest.java
index 8c5cad5..22d5a71 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobPlanInfoTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobPlanInfoTest.java
@@ -18,7 +18,8 @@
package org.apache.flink.runtime.rest.messages;
-import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
/**
* Tests that the {@link JobPlanInfo} can be marshalled and unmarshalled.
@@ -32,9 +33,7 @@ public class JobPlanInfoTest extends RestResponseMarshallingTestBase<JobPlanInfo
@Override
protected JobPlanInfo getTestResponseInstance() {
- JobID jobID = new JobID();
- String jobName = "job_007";
- String jsonPlan = "{\"jobid\":\"" + jobID + "\", \"name\":\"" + jobName + "\", \"nodes\":[]}";
- return new JobPlanInfo(jsonPlan);
+ JobGraph jg = new JobGraph("job_007");
+ return new JobPlanInfo(JsonPlanGenerator.generatePlan(jg));
}
}
[2/4] flink git commit: [FLINK-9810][rest] Close jar file in
JarListHandler
Posted by ch...@apache.org.
[FLINK-9810][rest] Close jar file in JarListHandler
This closes #6310.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2f7ab90b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2f7ab90b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2f7ab90b
Branch: refs/heads/release-1.5
Commit: 2f7ab90b1ae12bec1d31ffad93e3d2b44b425cfe
Parents: 7483b04
Author: zentol <ch...@apache.org>
Authored: Wed Jul 11 18:19:53 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Fri Jul 13 06:10:32 2018 +0200
----------------------------------------------------------------------
.../apache/flink/runtime/webmonitor/handlers/JarListHandler.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2f7ab90b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
index 7848478..c3e2336 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
@@ -103,8 +103,7 @@ public class JarListHandler extends AbstractRestHandler<RestfulGateway, EmptyReq
List<JarListInfo.JarEntryInfo> jarEntryList = new ArrayList<>();
String[] classes = new String[0];
- try {
- JarFile jar = new JarFile(f);
+ try (JarFile jar = new JarFile(f)) {
Manifest manifest = jar.getManifest();
String assemblerClass = null;
[4/4] flink git commit: [FLINK-9801][build] Add missing example
dependencies to flink-dist
Posted by ch...@apache.org.
[FLINK-9801][build] Add missing example dependencies to flink-dist
This closes #6304.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7483b04e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7483b04e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7483b04e
Branch: refs/heads/release-1.5
Commit: 7483b04ecaf2cde07af40fc6a9e30ef36b38c738
Parents: 8a564f8
Author: zentol <ch...@apache.org>
Authored: Wed Jul 11 12:23:24 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Fri Jul 13 06:10:32 2018 +0200
----------------------------------------------------------------------
flink-dist/pom.xml | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/7483b04e/flink-dist/pom.xml
----------------------------------------------------------------------
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 923f078..1118d43 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -195,6 +195,20 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-examples-streaming_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-gelly-examples_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>