You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ma...@apache.org on 2022/02/18 08:51:32 UTC
[flink] 02/02: [FLINK-26105][e2e] Fixes log file extension
This is an automated email from the ASF dual-hosted git repository.
mapohl pushed a commit to branch release-1.14
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 7cebc6cfe1b16a49e67841263b61039f094099a7
Author: Matthias Pohl <ma...@ververica.com>
AuthorDate: Wed Feb 16 16:16:12 2022 +0100
[FLINK-26105][e2e] Fixes log file extension
Rolling log file naming strategy is enabled by default which
might cause test instabilities do to the relevant substring
not being present in the *.log file. This change expands the
grep and find calls to also consider *.log.[0-9]+ files.
---
flink-end-to-end-tests/test-scripts/common.sh | 14 +++++++-------
flink-end-to-end-tests/test-scripts/common_ha.sh | 2 +-
.../test-scripts/queryable_state_base.sh | 4 ++--
flink-end-to-end-tests/test-scripts/test_cli.sh | 2 +-
.../test-scripts/test_rocksdb_state_memory_control.sh | 2 +-
tools/ci/controller_utils.sh | 2 +-
tools/ci/maven-utils.sh | 2 +-
7 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh
index 17e9a4e..1e5e51a 100644
--- a/flink-end-to-end-tests/test-scripts/common.sh
+++ b/flink-end-to-end-tests/test-scripts/common.sh
@@ -476,7 +476,7 @@ function wait_for_job_state_transition {
echo "Waiting for job ($job) to switch from state ${initial_state} to state ${next_state} ..."
while : ; do
- N=$(grep -o "($job) switched from state ${initial_state} to ${next_state}" $FLINK_LOG_DIR/*standalonesession*.log | tail -1)
+ N=$(grep -o "($job) switched from state ${initial_state} to ${next_state}" $FLINK_LOG_DIR/*standalonesession*.log* | tail -1)
if [[ -z $N ]]; then
sleep 1
@@ -511,7 +511,7 @@ function wait_job_terminal_state {
echo "Waiting for job ($job) to reach terminal state $expected_terminal_state ..."
while : ; do
- local N=$(grep -o "Job $job reached terminal state .*" $FLINK_LOG_DIR/*$log_file_name*.log | tail -1 || true)
+ local N=$(grep -o "Job $job reached terminal state .*" $FLINK_LOG_DIR/*$log_file_name*.log* | tail -1 || true)
if [[ -z $N ]]; then
sleep 1
else
@@ -633,7 +633,7 @@ function get_job_metric {
function get_metric_processed_records {
OPERATOR=$1
JOB_NAME="${2:-General purpose test job}"
- N=$(grep ".${JOB_NAME}.$OPERATOR.numRecordsIn:" $FLINK_LOG_DIR/*taskexecutor*.log | sed 's/.* //g' | tail -1)
+ N=$(grep ".${JOB_NAME}.$OPERATOR.numRecordsIn:" $FLINK_LOG_DIR/*taskexecutor*.log* | sed 's/.* //g' | tail -1)
if [ -z $N ]; then
N=0
fi
@@ -643,7 +643,7 @@ function get_metric_processed_records {
function get_num_metric_samples {
OPERATOR=$1
JOB_NAME="${2:-General purpose test job}"
- N=$(grep ".${JOB_NAME}.$OPERATOR.numRecordsIn:" $FLINK_LOG_DIR/*taskexecutor*.log | wc -l)
+ N=$(grep ".${JOB_NAME}.$OPERATOR.numRecordsIn:" $FLINK_LOG_DIR/*taskexecutor*.log* | wc -l)
if [ -z $N ]; then
N=0
fi
@@ -693,7 +693,7 @@ function wait_num_of_occurence_in_logs {
echo "Waiting for text ${text} to appear ${number} of times in logs..."
while : ; do
- N=$(grep -o "${text}" $FLINK_LOG_DIR/*${logs}*.log | wc -l)
+ N=$(grep -o "${text}" $FLINK_LOG_DIR/*${logs}*.log* | wc -l)
if [ -z $N ]; then
N=0
@@ -722,7 +722,7 @@ function wait_num_checkpoints {
echo "Waiting for job ($JOB) to have at least $NUM_CHECKPOINTS completed checkpoints ..."
while : ; do
- N=$(grep -o "Completed checkpoint [1-9]* for job $JOB" $FLINK_LOG_DIR/*standalonesession*.log | awk '{print $3}' | tail -1)
+ N=$(grep -o "Completed checkpoint [1-9]* for job $JOB" $FLINK_LOG_DIR/*standalonesession*.log* | awk '{print $3}' | tail -1)
if [ -z $N ]; then
N=0
@@ -764,7 +764,7 @@ function expect_in_taskmanager_logs {
local expected="$1"
local timeout=$2
local i=0
- local logfile="$FLINK_LOG_DIR/flink*taskexecutor*log"
+ local logfile="$FLINK_LOG_DIR/flink*taskexecutor*log*"
while ! grep "${expected}" ${logfile} > /dev/null; do
diff --git a/flink-end-to-end-tests/test-scripts/common_ha.sh b/flink-end-to-end-tests/test-scripts/common_ha.sh
index 4612fec..23f89ee 100644
--- a/flink-end-to-end-tests/test-scripts/common_ha.sh
+++ b/flink-end-to-end-tests/test-scripts/common_ha.sh
@@ -49,7 +49,7 @@ function verify_num_occurences_in_logs() {
local text="$2"
local expected_no="$3"
- local actual_no=$(grep -r --include "*${log_pattern}*.log" -e "${text}" "$FLINK_LOG_DIR/" | cut -d ":" -f 1 | uniq | wc -l)
+ local actual_no=$(grep -r --include "*${log_pattern}*.log*" -e "${text}" "$FLINK_LOG_DIR/" | cut -d ":" -f 1 | sed "s/\.[0-9]\{1,\}$//g" | uniq | wc -l)
[[ "${expected_no}" -eq "${actual_no}" ]]
}
diff --git a/flink-end-to-end-tests/test-scripts/queryable_state_base.sh b/flink-end-to-end-tests/test-scripts/queryable_state_base.sh
index 87957ec..9825931 100644
--- a/flink-end-to-end-tests/test-scripts/queryable_state_base.sh
+++ b/flink-end-to-end-tests/test-scripts/queryable_state_base.sh
@@ -25,7 +25,7 @@ function link_queryable_state_lib {
# Returns the ip address of the queryable state server
function get_queryable_state_server_ip {
- local ip=$(cat $FLINK_LOG_DIR/flink*taskexecutor*log \
+ local ip=$(cat $FLINK_LOG_DIR/flink*taskexecutor*log* \
| grep "Started Queryable State Server" \
| head -1 \
| grep -Eo "\.*[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.*")
@@ -35,7 +35,7 @@ function get_queryable_state_server_ip {
# Returns the ip address of the queryable state server
function get_queryable_state_proxy_port {
- local port=$(cat $FLINK_LOG_DIR/flink*taskexecutor*log \
+ local port=$(cat $FLINK_LOG_DIR/flink*taskexecutor*log* \
| grep "Started Queryable State Proxy Server" \
| head -1 \
| grep -Eo "\.*\:([0-9]{3,5})\.*" | tr -d ":.")
diff --git a/flink-end-to-end-tests/test-scripts/test_cli.sh b/flink-end-to-end-tests/test-scripts/test_cli.sh
index 985ce67..0746469 100755
--- a/flink-end-to-end-tests/test-scripts/test_cli.sh
+++ b/flink-end-to-end-tests/test-scripts/test_cli.sh
@@ -65,7 +65,7 @@ function extract_valid_job_list_by_type_from_job_list_return() {
}
function extract_task_manager_slot_request_count() {
- COUNT=`grep "Receive slot request" $FLINK_LOG_DIR/*taskexecutor*.log | wc -l`
+ COUNT=`grep "Receive slot request" $FLINK_LOG_DIR/*taskexecutor*.log* | wc -l`
echo $COUNT
}
diff --git a/flink-end-to-end-tests/test-scripts/test_rocksdb_state_memory_control.sh b/flink-end-to-end-tests/test-scripts/test_rocksdb_state_memory_control.sh
index 66bfa43..d42e806 100755
--- a/flink-end-to-end-tests/test-scripts/test_rocksdb_state_memory_control.sh
+++ b/flink-end-to-end-tests/test-scripts/test_rocksdb_state_memory_control.sh
@@ -80,7 +80,7 @@ function buildBaseJobCmd {
function find_max_block_cache_usage() {
OPERATOR=$1
JOB_NAME="${2:-General purpose test job}"
- N=$(grep ".${JOB_NAME}.$OPERATOR.rocksdb.block-cache-usage:" $FLINK_LOG_DIR/*taskexecutor*.log | sed 's/.* //g' | sort -rn | head -n 1)
+ N=$(grep ".${JOB_NAME}.$OPERATOR.rocksdb.block-cache-usage:" $FLINK_LOG_DIR/*taskexecutor*.log* | sed 's/.* //g' | sort -rn | head -n 1)
if [ -z $N ]; then
N=0
fi
diff --git a/tools/ci/controller_utils.sh b/tools/ci/controller_utils.sh
index da0a414..8700379 100644
--- a/tools/ci/controller_utils.sh
+++ b/tools/ci/controller_utils.sh
@@ -33,7 +33,7 @@ print_system_info() {
# locate YARN logs and put them into artifacts directory
put_yarn_logs_to_artifacts() {
- for file in `find ./flink-yarn-tests/target -type f -name '*.log' -or -name '*.out'`; do
+ for file in `find ./flink-yarn-tests/target -type f -name '*.log*' -or -name '*.out'`; do
TARGET_FILE=`echo "$file" | grep -Eo "container_[0-9_]+/(.*).[a-z]{3}"`
TARGET_DIR=`dirname "$TARGET_FILE"`
mkdir -p "$DEBUG_FILES_OUTPUT_DIR/yarn-tests/$TARGET_DIR"
diff --git a/tools/ci/maven-utils.sh b/tools/ci/maven-utils.sh
index caea57b..c6b0491 100755
--- a/tools/ci/maven-utils.sh
+++ b/tools/ci/maven-utils.sh
@@ -73,7 +73,7 @@ function collect_coredumps {
local SEARCHDIR=$1
local TARGET_DIR=$2
echo "Searching for .dump, .dumpstream and related files in '$SEARCHDIR'"
- for file in `find $SEARCHDIR -type f -regextype posix-extended -iregex '.*\.hprof|.*\.dump|.*\.dumpstream|.*hs.*\.log|.*/core(.[0-9]+)?$'`; do
+ for file in `find $SEARCHDIR -type f -regextype posix-extended -iregex '.*\.hprof|.*\.dump|.*\.dumpstream|.*hs.*\.log(\.[0-9]+)?|.*/core(\.[0-9]+)?$'`; do
echo "Moving '$file' to target directory ('$TARGET_DIR')"
mv $file $TARGET_DIR/$(echo $file | tr "/" "-")
done