You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2021/05/11 08:23:53 UTC
[flink] branch release-1.13 updated (b5f375d -> b5a9336)
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a change to branch release-1.13
in repository https://gitbox.apache.org/repos/asf/flink.git.
from b5f375d [FLINK-22525][table-api] Fix gmt format in Flink from GMT-8:00 to GMT-08:00 (#15874)
new 2502e40 [hotfix][test] Adds -e flag to interpret newline in the right way
new b5a9336 [FLINK-22566][test] Adds log extraction for the worker nodes
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../test-scripts/common_yarn_docker.sh | 46 +++++++++++++++-------
1 file changed, 32 insertions(+), 14 deletions(-)
[flink] 02/02: [FLINK-22566][test] Adds log extraction for the
worker nodes
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch release-1.13
in repository https://gitbox.apache.org/repos/asf/flink.git
commit b5a9336b9b93920708a6ca4126d4035bb165bab5
Author: Matthias Pohl <ma...@ververica.com>
AuthorDate: Fri May 7 14:53:07 2021 +0200
[FLINK-22566][test] Adds log extraction for the worker nodes
We struggled to get the logs of the node manager which made it hard to
investigate FLINK-22566 where there was a lag between setting up the YARN
containers and starting the TaskExecutor. Hopefully, the nodemanager logs
located on the worker nodes will help next time to investigate something like
that.
---
.../test-scripts/common_yarn_docker.sh | 40 ++++++++++++++++------
1 file changed, 29 insertions(+), 11 deletions(-)
diff --git a/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh b/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
index 95d3882..f640976 100755
--- a/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
+++ b/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
@@ -144,19 +144,13 @@ function debug_copy_and_show_logs {
echo -e "\n\nCurrently running JVMs"
jps -v
- echo -e "\n\nHadoop logs:"
- mkdir -p $TEST_DATA_DIR/logs
- docker cp master:/var/log/hadoop/ $TEST_DATA_DIR/logs/
- ls -lisah $TEST_DATA_DIR/logs/hadoop
- for f in $TEST_DATA_DIR/logs/hadoop/*; do
- echo "$f:"
- cat $f
- done
+ local log_directory="$TEST_DATA_DIR/logs"
+ local yarn_docker_containers="master $(docker ps --format '{{.Names}}' | grep worker)"
- echo -e "\n\nDocker logs:"
- docker logs master
+ extract_hadoop_logs ${log_directory} ${yarn_docker_containers}
+ print_logs ${log_directory}
- echo -e "\n\nFlink logs:"
+ echo -e "\n\n ==== Flink logs ===="
docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user"
docker exec master bash -c "yarn application -list -appStates ALL"
application_id=`docker exec master bash -c "yarn application -list -appStates ALL" | grep -i "Flink" | grep -i "cluster" | awk '{print \$1}'`
@@ -167,6 +161,30 @@ function debug_copy_and_show_logs {
docker exec master bash -c "kdestroy"
}
+function extract_hadoop_logs() {
+ local parent_folder="$1"
+ shift
+ docker_container_aliases="$@"
+
+ for docker_container_alias in $(echo ${docker_container_aliases}); do
+ local target_container_log_folder="${parent_folder}/${docker_container_alias}"
+ echo "Extracting ${docker_container_alias} Hadoop logs into ${target_container_log_folder}"
+ mkdir -p "${target_container_log_folder}"
+ docker cp "${docker_container_alias}:/var/log/hadoop/" "${target_container_log_folder}"
+
+ local target_container_docker_log_file="${target_container_log_folder}/docker-${docker_container_alias}.log"
+ echo "Extracting ${docker_container_alias} Docker logs into ${target_container_docker_log_file}"
+ docker logs "${docker_container_alias}" > "${target_container_docker_log_file}"
+ done
+}
+
+function print_logs() {
+ local parent_folder="$1"
+
+ ls -lisahR "${parent_folder}"
+ find "${parent_folder}" -type f -exec echo -e "\n\nContent of {}:" \; -exec cat {} \;
+}
+
# expects only one application to be running and waits until this one is in
# final state SUCCEEDED
function wait_for_single_yarn_application {
[flink] 01/02: [hotfix][test] Adds -e flag to interpret newline in
the right way
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch release-1.13
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 2502e40d56bbcda330ca60477c3fc965bd53d15a
Author: Matthias Pohl <ma...@ververica.com>
AuthorDate: Fri May 7 13:42:44 2021 +0200
[hotfix][test] Adds -e flag to interpret newline in the right way
---
flink-end-to-end-tests/test-scripts/common_yarn_docker.sh | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh b/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
index a4ffca7..95d3882 100755
--- a/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
+++ b/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
@@ -138,13 +138,13 @@ END
function debug_copy_and_show_logs {
echo "Debugging failed YARN Docker test:"
- echo "\nCurrently running containers"
+ echo -e "\nCurrently running containers"
docker ps
- echo "\n\nCurrently running JVMs"
+ echo -e "\n\nCurrently running JVMs"
jps -v
- echo "\n\nHadoop logs:"
+ echo -e "\n\nHadoop logs:"
mkdir -p $TEST_DATA_DIR/logs
docker cp master:/var/log/hadoop/ $TEST_DATA_DIR/logs/
ls -lisah $TEST_DATA_DIR/logs/hadoop
@@ -153,15 +153,15 @@ function debug_copy_and_show_logs {
cat $f
done
- echo "\n\nDocker logs:"
+ echo -e "\n\nDocker logs:"
docker logs master
- echo "\n\nFlink logs:"
+ echo -e "\n\nFlink logs:"
docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user"
docker exec master bash -c "yarn application -list -appStates ALL"
application_id=`docker exec master bash -c "yarn application -list -appStates ALL" | grep -i "Flink" | grep -i "cluster" | awk '{print \$1}'`
- echo "Application ID: '$application_id'"
+ echo -e "\n\nApplication ID: '$application_id'"
docker exec master bash -c "yarn logs -applicationId $application_id"
docker exec master bash -c "kdestroy"