You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/01/24 12:32:13 UTC
[1/4] flink git commit: [FLINK-8473][webUI] Improve error behavior of
JarListHandler
Repository: flink
Updated Branches:
refs/heads/release-1.4 df0526172 -> c701a335b
[FLINK-8473][webUI] Improve error behavior of JarListHandler
This closes #5331.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/20be204b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/20be204b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/20be204b
Branch: refs/heads/release-1.4
Commit: 20be204b96edd5c92683013a4c5af9ea4096acca
Parents: df05261
Author: zentol <ch...@apache.org>
Authored: Mon Jan 22 13:29:34 2018 +0100
Committer: zentol <ch...@apache.org>
Committed: Wed Jan 24 10:42:16 2018 +0100
----------------------------------------------------------------------
.../runtime/webmonitor/HttpRequestHandler.java | 4 +++
.../runtime/webmonitor/WebRuntimeMonitor.java | 32 +++++++++++++++-----
.../webmonitor/handlers/JarActionHandler.java | 11 +++++++
.../webmonitor/handlers/JarDeleteHandler.java | 13 ++++++++
.../webmonitor/handlers/JarListHandler.java | 11 +++++++
5 files changed, 64 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/20be204b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/HttpRequestHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/HttpRequestHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/HttpRequestHandler.java
index f8b51d7..a0fda9d 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/HttpRequestHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/HttpRequestHandler.java
@@ -133,6 +133,10 @@ public class HttpRequestHandler extends SimpleChannelInboundHandler<HttpObject>
String name = file.getFilename();
File target = new File(tmpDir, UUID.randomUUID() + "_" + name);
+ if (!tmpDir.exists()) {
+ WebRuntimeMonitor.logExternalUploadDirDeletion(tmpDir);
+ WebRuntimeMonitor.checkAndCreateUploadDir(tmpDir);
+ }
file.renameTo(target);
QueryStringEncoder encoder = new QueryStringEncoder(currentRequestPath);
http://git-wip-us.apache.org/repos/asf/flink/blob/20be204b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
index fe5f106..e7bb157 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
@@ -188,13 +188,7 @@ public class WebRuntimeMonitor implements WebMonitor {
if (webSubmitAllow) {
// create storage for uploads
this.uploadDir = getUploadDir(config);
- // the upload directory should either 1. exist and writable or 2. can be created and writable
- if (!(uploadDir.exists() && uploadDir.canWrite()) && !(uploadDir.mkdirs() && uploadDir.canWrite())) {
- throw new IOException(
- String.format("Jar upload directory %s cannot be created or is not writable.",
- uploadDir.getAbsolutePath()));
- }
- LOG.info("Using directory {} for web frontend JAR file uploads", uploadDir);
+ checkAndCreateUploadDir(uploadDir);
}
else {
this.uploadDir = null;
@@ -578,4 +572,28 @@ public class WebRuntimeMonitor implements WebMonitor {
boolean uploadDirSpecified = configuration.contains(WebOptions.UPLOAD_DIR);
return uploadDirSpecified ? baseDir : new File(baseDir, "flink-web-" + UUID.randomUUID());
}
+
+ public static void logExternalUploadDirDeletion(File uploadDir) {
+ LOG.warn("Jar storage directory {} has been deleted externally. Previously uploaded jars are no longer available.", uploadDir.getAbsolutePath());
+ }
+
+ /**
+ * Checks whether the given directory exists and is writable. If it doesn't exist this method will attempt to create
+ * it.
+ *
+ * @param uploadDir directory to check
+ * @throws IOException if the directory does not exist and cannot be created, or if the directory isn't writable
+ */
+ public static synchronized void checkAndCreateUploadDir(File uploadDir) throws IOException {
+ if (uploadDir.exists() && uploadDir.canWrite()) {
+ LOG.info("Using directory {} for web frontend JAR file uploads.", uploadDir);
+ } else if (uploadDir.mkdirs() && uploadDir.canWrite()) {
+ LOG.info("Created directory {} for web frontend JAR file uploads.", uploadDir);
+ } else {
+ LOG.warn("Jar upload directory {} cannot be created or is not writable.", uploadDir.getAbsolutePath());
+ throw new IOException(
+ String.format("Jar upload directory %s cannot be created or is not writable.",
+ uploadDir.getAbsolutePath()));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/20be204b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandler.java
index c601a8d..eea3a87 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandler.java
@@ -37,11 +37,13 @@ import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
+import org.apache.flink.runtime.webmonitor.WebRuntimeMonitor;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
import java.io.File;
+import java.io.IOException;
import java.io.StringWriter;
import java.net.URISyntaxException;
import java.net.URL;
@@ -66,6 +68,15 @@ public abstract class JarActionHandler extends AbstractJsonRequestHandler {
// generate the graph
JobGraph graph = null;
+ if (!jarDir.exists()) {
+ WebRuntimeMonitor.logExternalUploadDirDeletion(jarDir);
+ try {
+ WebRuntimeMonitor.checkAndCreateUploadDir(jarDir);
+ } catch (IOException ioe) {
+ // the following code will throw an exception since the jar can't be found
+ }
+ }
+
PackagedProgram program = new PackagedProgram(
new File(jarDir, config.getJarFile()),
config.getEntryClass(),
http://git-wip-us.apache.org/repos/asf/flink/blob/20be204b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandler.java
index fb7fb40..c1a5f60 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandler.java
@@ -21,12 +21,14 @@ package org.apache.flink.runtime.webmonitor.handlers;
import org.apache.flink.runtime.jobmaster.JobManagerGateway;
import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
+import org.apache.flink.runtime.webmonitor.WebRuntimeMonitor;
import org.apache.flink.util.FlinkException;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
import java.io.File;
import java.io.FilenameFilter;
+import java.io.IOException;
import java.io.StringWriter;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
@@ -64,6 +66,17 @@ public class JarDeleteHandler extends AbstractJsonRequestHandler {
return name.equals(file);
}
});
+
+ if (list == null) {
+ WebRuntimeMonitor.logExternalUploadDirDeletion(jarDir);
+ try {
+ WebRuntimeMonitor.checkAndCreateUploadDir(jarDir);
+ } catch (IOException ioe) {
+ // entire directory doesn't exist anymore, continue as if deletion succeeded
+ }
+ list = new File[0];
+ }
+
boolean success = false;
for (File f: list) {
// although next to impossible for multiple files, we still delete them.
http://git-wip-us.apache.org/repos/asf/flink/blob/20be204b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
index 59a436f..2b56ecd 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
@@ -23,6 +23,7 @@ import org.apache.flink.runtime.jobmaster.JobManagerGateway;
import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
import org.apache.flink.runtime.webmonitor.RuntimeMonitorHandler;
+import org.apache.flink.runtime.webmonitor.WebRuntimeMonitor;
import org.apache.flink.util.FlinkException;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
@@ -77,6 +78,16 @@ public class JarListHandler extends AbstractJsonRequestHandler {
}
});
+ if (list == null) {
+ WebRuntimeMonitor.logExternalUploadDirDeletion(jarDir);
+ try {
+ WebRuntimeMonitor.checkAndCreateUploadDir(jarDir);
+ } catch (IOException ioe) {
+ // re-throwing an exception here breaks the UI
+ }
+ list = new File[0];
+ }
+
// last modified ascending order
Arrays.sort(list, (f1, f2) -> Long.compare(f2.lastModified(), f1.lastModified()));
[3/4] flink git commit: [FLINK-8433] [doc] Remove ununsed
CheckpointedRestoring interface
Posted by ch...@apache.org.
[FLINK-8433] [doc] Remove ununsed CheckpointedRestoring interface
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/77df20bc
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/77df20bc
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/77df20bc
Branch: refs/heads/release-1.4
Commit: 77df20bc50322166a517cb293abfbf35320586a1
Parents: e4d7ea2
Author: zhangminglei <zm...@163.com>
Authored: Mon Jan 15 17:37:41 2018 +0800
Committer: zentol <ch...@apache.org>
Committed: Wed Jan 24 10:50:29 2018 +0100
----------------------------------------------------------------------
docs/dev/stream/state/state.md | 9 +--------
1 file changed, 1 insertion(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/77df20bc/docs/dev/stream/state/state.md
----------------------------------------------------------------------
diff --git a/docs/dev/stream/state/state.md b/docs/dev/stream/state/state.md
index 9956998..ce1ce82 100644
--- a/docs/dev/stream/state/state.md
+++ b/docs/dev/stream/state/state.md
@@ -331,8 +331,7 @@ the basic even-split redistribution list state:
{% highlight java %}
public class BufferingSink
implements SinkFunction<Tuple2<String, Integer>>,
- CheckpointedFunction,
- CheckpointedRestoring<ArrayList<Tuple2<String, Integer>>> {
+ CheckpointedFunction {
private final int threshold;
@@ -379,12 +378,6 @@ public class BufferingSink
}
}
}
-
- @Override
- public void restoreState(ArrayList<Tuple2<String, Integer>> state) throws Exception {
- // this is from the CheckpointedRestoring interface.
- this.bufferedElements.addAll(state);
- }
}
{% endhighlight %}
</div>
[2/4] flink git commit: [FLINK-8079][tests] Stop end-to-end test
execution after first failure
Posted by ch...@apache.org.
[FLINK-8079][tests] Stop end-to-end test execution after first failure
This closes #5156.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e4d7ea26
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e4d7ea26
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e4d7ea26
Branch: refs/heads/release-1.4
Commit: e4d7ea264125322a68ea05ca141eef18a3016846
Parents: 20be204
Author: zentol <ch...@apache.org>
Authored: Tue Dec 12 14:04:34 2017 +0100
Committer: zentol <ch...@apache.org>
Committed: Wed Jan 24 10:42:22 2018 +0100
----------------------------------------------------------------------
tools/travis_mvn_watchdog.sh | 71 ++++++++++++++++++++++-----------------
1 file changed, 41 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e4d7ea26/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 42f7ae3..5edeb89 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -519,8 +519,9 @@ case $TEST in
(connectors)
if [ $EXIT_CODE == 0 ]; then
check_shaded_artifacts_s3_fs hadoop
+ EXIT_CODE=$(($EXIT_CODE+$?))
check_shaded_artifacts_s3_fs presto
- EXIT_CODE=$?
+ EXIT_CODE=$(($EXIT_CODE+$?))
else
echo "=============================================================================="
echo "Compilation/test failure detected, skipping shaded dependency check."
@@ -543,35 +544,45 @@ case $TEST in
printf "Running end-to-end tests\n"
printf "==============================================================================\n"
- printf "\n==============================================================================\n"
- printf "Running Wordcount end-to-end test\n"
- printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_batch_wordcount.sh
- EXIT_CODE=$(($EXIT_CODE+$?))
-
- printf "\n==============================================================================\n"
- printf "Running Kafka end-to-end test\n"
- printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_streaming_kafka010.sh
- EXIT_CODE=$(($EXIT_CODE+$?))
-
- printf "\n==============================================================================\n"
- printf "Running class loading end-to-end test\n"
- printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_streaming_classloader.sh
- EXIT_CODE=$(($EXIT_CODE+$?))
-
- printf "\n==============================================================================\n"
- printf "Running Shaded Hadoop S3A end-to-end test\n"
- printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_shaded_hadoop_s3a.sh
- EXIT_CODE=$(($EXIT_CODE+$?))
-
- printf "\n==============================================================================\n"
- printf "Running Shaded Presto S3 end-to-end test\n"
- printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_shaded_presto_s3.sh
- EXIT_CODE=$(($EXIT_CODE+$?))
+ if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Wordcount end-to-end test\n"
+ printf "==============================================================================\n"
+ FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_batch_wordcount.sh
+ EXIT_CODE=$?
+ fi
+
+ if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Kafka end-to-end test\n"
+ printf "==============================================================================\n"
+ FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_streaming_kafka010.sh
+ EXIT_CODE=$?
+ fi
+
+ if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running class loading end-to-end test\n"
+ printf "==============================================================================\n"
+ FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_streaming_classloader.sh
+ EXIT_CODE=$?
+ fi
+
+ if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Shaded Hadoop S3A end-to-end test\n"
+ printf "==============================================================================\n"
+ FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_shaded_hadoop_s3a.sh
+ EXIT_CODE=$?
+ fi
+
+ if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Shaded Presto S3 end-to-end test\n"
+ printf "==============================================================================\n"
+ FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_shaded_presto_s3.sh
+ EXIT_CODE=$?
+ fi
else
printf "\n==============================================================================\n"
printf "Previous build failure detected, skipping end-to-end tests.\n"
[4/4] flink git commit: [hotfix] [build] Print cache info
Posted by ch...@apache.org.
[hotfix] [build] Print cache info
Print the size of the Maven cache copied for each TravisCI job.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c701a335
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c701a335
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c701a335
Branch: refs/heads/release-1.4
Commit: c701a335bef4caa9745498d1505e6ef46942d764
Parents: 77df20b
Author: Greg Hogan <co...@greghogan.com>
Authored: Wed Jan 10 14:07:57 2018 -0500
Committer: zentol <ch...@apache.org>
Committed: Wed Jan 24 10:52:18 2018 +0100
----------------------------------------------------------------------
tools/travis_mvn_watchdog.sh | 41 ++++++++++++++++++++++-----------------
1 file changed, 23 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c701a335/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 5edeb89..7c27392 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -186,6 +186,28 @@ ARTIFACTS_FILE=${TRAVIS_JOB_NUMBER}.tar.gz
# FUNCTIONS
# =============================================================================
+print_system_info() {
+ FOLD_ESCAPE="\x0d\x1b"
+ COLOR_ON="\x5b\x30\x4b\x1b\x5b\x33\x33\x3b\x31\x6d"
+ COLOR_OFF="\x1b\x5b\x30\x6d"
+
+ echo -e "travis_fold:start:cpu_info${FOLD_ESCAPE}${COLOR_ON}CPU information${COLOR_OFF}"
+ lscpu
+ echo -en "travis_fold:end:cpu_info${FOLD_ESCAPE}"
+
+ echo -e "travis_fold:start:mem_info${FOLD_ESCAPE}${COLOR_ON}Memory information${COLOR_OFF}"
+ cat /proc/meminfo
+ echo -en "travis_fold:end:mem_info${FOLD_ESCAPE}"
+
+ echo -e "travis_fold:start:disk_info${FOLD_ESCAPE}${COLOR_ON}Disk information${COLOR_OFF}"
+ df -hH
+ echo -en "travis_fold:end:disk_info${FOLD_ESCAPE}"
+
+ echo -e "travis_fold:start:cache_info${FOLD_ESCAPE}${COLOR_ON}Cache information${COLOR_OFF}"
+ du -s --si $HOME/.m2
+ echo -en "travis_fold:end:cache_info${FOLD_ESCAPE}"
+}
+
upload_artifacts_s3() {
echo "PRODUCED build artifacts."
@@ -436,24 +458,7 @@ WD_PID=$!
echo "STARTED watchdog (${WD_PID})."
-
-# Print and fold CPU, memory, and filesystem info
-FOLD_ESCAPE="\x0d\x1b"
-COLOR_ON="\x5b\x30\x4b\x1b\x5b\x33\x33\x3b\x31\x6d"
-COLOR_OFF="\x1b\x5b\x30\x6d"
-
-echo -e "travis_fold:start:cpu_info${FOLD_ESCAPE}${COLOR_ON}CPU information${COLOR_OFF}"
-lscpu
-echo -en "travis_fold:end:cpu_info${FOLD_ESCAPE}"
-
-echo -e "travis_fold:start:mem_info${FOLD_ESCAPE}${COLOR_ON}Memory information${COLOR_OFF}"
-cat /proc/meminfo
-echo -en "travis_fold:end:mem_info${FOLD_ESCAPE}"
-
-echo -e "travis_fold:start:disk_info${FOLD_ESCAPE}${COLOR_ON}Disk information${COLOR_OFF}"
-df -hH
-echo -en "travis_fold:end:disk_info${FOLD_ESCAPE}"
-
+print_system_info
# Make sure to be in project root
cd $HERE/../