You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by he...@apache.org on 2026/04/21 10:03:27 UTC
(doris) branch master updated: [fix](docker) Stabilize external docker startup (#62664)
This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7554ffb5acc [fix](docker) Stabilize external docker startup (#62664)
7554ffb5acc is described below
commit 7554ffb5accc7d53cd79577258c20c0d42a78828
Author: Socrates <su...@selectdb.com>
AuthorDate: Tue Apr 21 18:03:19 2026 +0800
[fix](docker) Stabilize external docker startup (#62664)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #62103
Problem Summary:
External regression docker startup became unstable after the Hive docker
startup workflow was optimized. The startup script starts many heavy
thirdparty components and depends on `docker compose --wait`; when a
component fails or stays in health-check startup, the top-level log did
not clearly show which component was still blocking the run. When one
component failed, remaining background startup jobs and compose stacks
could continue changing container state, making retry and log collection
unstable.
This change also fixes concrete regressions observed in the external
regression logs and on the pipeline host:
- HiveServer2 did not load the same Hive aux jars as the metastore, so
LZO table DDL could fail with `Cannot find class
'com.hadoop.mapreduce.LzoTextInputFormat'`.
- HiveServer2 health checks could hang in `docker compose --wait`
because HS2 was started as root and `beeline -n health_check` failed
with `User: root is not allowed to impersonate health_check`. The
generated Hive config now disables HS2 doAs for these docker test
services.
- The Iceberg stack used generic container names such as
`${CONTAINER_UID}postgres`, which could conflict with the standalone
PostgreSQL external component.
- DB2 can take longer than the old health window on busy external
pipeline hosts.
The fix launches all selected external docker compose startup jobs
immediately, prints periodic progress for startup jobs that are still
running, cleans registered compose stacks after startup failure, loads
Hive aux jars before starting HiveServer2, aligns HiveServer2 docker
auth behavior with the health check, and gives Iceberg-internal
containers unique names.
---
.../thirdparties/docker-compose/db2/db2.yaml.tpl | 3 +-
.../docker-compose/hive/hadoop-hive.env.tpl | 1 +
.../docker-compose/hive/hive-2x.yaml.tpl | 2 +-
.../docker-compose/hive/hive-3x.yaml.tpl | 2 +-
.../docker-compose/hive/scripts/hive-module-lib.sh | 5 +-
.../hive/scripts/start-hive-server.sh | 27 ++++
.../docker-compose/iceberg/iceberg.yaml.tpl | 6 +-
docker/thirdparties/run-thirdparties-docker.sh | 164 ++++++++++++++++++---
8 files changed, 182 insertions(+), 28 deletions(-)
diff --git a/docker/thirdparties/docker-compose/db2/db2.yaml.tpl b/docker/thirdparties/docker-compose/db2/db2.yaml.tpl
index 76bf79373c6..49a09656549 100644
--- a/docker/thirdparties/docker-compose/db2/db2.yaml.tpl
+++ b/docker/thirdparties/docker-compose/db2/db2.yaml.tpl
@@ -27,7 +27,8 @@ services:
test: ["CMD-SHELL", "su - db2inst1 -c \"source ~/.bash_profile; db2 connect to doris && db2 'select 1 from sysibm.sysdummy1'\""]
interval: 20s
timeout: 60s
- retries: 10
+ retries: 30
+ start_period: 300s
volumes:
- ./init:/docker-entrypoint-initdb.d
environment:
diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
index ca69b128662..55c67c82c7a 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
@@ -25,6 +25,7 @@ HIVE_SITE_CONF_hive_metastore_uris=thrift://${HIVE_HOST_ALIAS}:${HMS_PORT}
HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT}
HIVE_SITE_CONF_hive_server2_webui_port=0
+HIVE_SITE_CONF_hive_server2_enable_doAs=false
HIVE_SITE_CONF_hive_compactor_initiator_on=true
HIVE_SITE_CONF_hive_compactor_worker_threads=2
HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index 4c39acd42ba..f6829247627 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -67,10 +67,10 @@ services:
hostname: ${HIVE_HOST_ALIAS}
env_file:
- ./hadoop-hive-2x.env
+ command: /bin/bash /mnt/scripts/start-hive-server.sh
environment:
HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${IP_HOST}:${PG_PORT}/metastore"
SERVICE_PRECONDITION: "${IP_HOST}:${HMS_PORT}"
- HIVE_SITE_CONF_hive_aux_jars_path: "file:///mnt/scripts/auxlib/json-serde-1.3.9-SNAPSHOT-jar-with-dependencies.jar"
container_name: ${CONTAINER_UID}hive2-server
expose:
- "${HS_PORT}"
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index a6dc44941a2..6bd9bbd7e6f 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -68,11 +68,11 @@ services:
hostname: ${HIVE_HOST_ALIAS}
env_file:
- ./hadoop-hive-3x.env
+ command: /bin/bash /mnt/scripts/start-hive-server.sh
environment:
HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${IP_HOST}:${PG_PORT}/metastore"
SERVICE_PRECONDITION: "${IP_HOST}:${HMS_PORT}"
JVM_OPTS: -Xmx2g
- HIVE_SITE_CONF_hive_aux_jars_path: "file:///mnt/scripts/auxlib/json-serde-1.3.9-SNAPSHOT-jar-with-dependencies.jar"
container_name: ${CONTAINER_UID}hive3-server
expose:
- "${HS_PORT}"
diff --git a/docker/thirdparties/docker-compose/hive/scripts/hive-module-lib.sh b/docker/thirdparties/docker-compose/hive/scripts/hive-module-lib.sh
index d57d5bcda7d..1cd00099c9b 100644
--- a/docker/thirdparties/docker-compose/hive/scripts/hive-module-lib.sh
+++ b/docker/thirdparties/docker-compose/hive/scripts/hive-module-lib.sh
@@ -24,6 +24,7 @@ set -eo pipefail
BOOTSTRAP_GROUPS="$(bootstrap_normalize_groups "${HIVE_BOOTSTRAP_GROUPS:-}")"
DEFAULT_MODULES=(default multi_catalog partition_type statistics tvf regression test preinstalled_hql view)
LAST_REFRESH_DETAIL=""
+HIVE_HQL_PARALLEL="${HIVE_HQL_PARALLEL:-${LOAD_PARALLEL}}"
ensure_hive_state_layout
@@ -276,8 +277,8 @@ refresh_preinstalled_hql_module() {
LAST_REFRESH_DETAIL="files=${#hqls_to_refresh[@]}($(format_refresh_preview 5 "${refresh_rel_paths[@]}"))"
# Phase 2 (parallel): execute changed files via xargs -P
- echo " [preinstalled_hql] refreshing ${#hqls_to_refresh[@]} files (parallel=${LOAD_PARALLEL})"
- printf '%s\0' "${hqls_to_refresh[@]}" | stdbuf -oL -eL xargs -0 -P "${LOAD_PARALLEL}" -I {} \
+ echo " [preinstalled_hql] refreshing ${#hqls_to_refresh[@]} files (parallel=${HIVE_HQL_PARALLEL})"
+ printf '%s\0' "${hqls_to_refresh[@]}" | stdbuf -oL -eL xargs -0 -P "${HIVE_HQL_PARALLEL}" -I {} \
stdbuf -oL -eL bash --noprofile --norc -ec '
hql_path="{}"
. /mnt/scripts/hive-module-lib.sh
diff --git a/docker/thirdparties/docker-compose/hive/scripts/start-hive-server.sh b/docker/thirdparties/docker-compose/hive/scripts/start-hive-server.sh
new file mode 100644
index 00000000000..90aff25275b
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/start-hive-server.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+if [[ "${HIVE_DEBUG:-0}" == "1" ]]; then
+ set -x
+fi
+
+. /mnt/scripts/hive-common-lib.sh
+
+prepare_hive_aux_lib
+exec /opt/hive/bin/hive --service hiveserver2
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
index 83c1ee6d031..0c4e8e1cc65 100644
--- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
@@ -53,7 +53,7 @@ services:
postgres:
image: postgis/postgis:14-3.3
- container_name: doris--postgres
+ container_name: doris--iceberg-postgres
environment:
POSTGRES_PASSWORD: 123456
POSTGRES_USER: root
@@ -101,7 +101,7 @@ services:
minio:
image: minio/minio:RELEASE.2025-01-20T14-49-07Z
- container_name: doris--minio
+ container_name: doris--iceberg-minio
ports:
- ${MINIO_API_PORT}:9000
healthcheck:
@@ -127,7 +127,7 @@ services:
minio:
condition: service_healthy
image: minio/mc:RELEASE.2025-01-17T23-25-50Z
- container_name: doris--mc
+ container_name: doris--iceberg-mc
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index 2ab7bb49734..2942b061bf6 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -57,6 +57,9 @@ STOP=0
NEED_RESERVE_PORTS=0
export NEED_LOAD_DATA=1
export LOAD_PARALLEL=$(( $(getconf _NPROCESSORS_ONLN) / 2 ))
+export START_PROGRESS_INTERVAL="${START_PROGRESS_INTERVAL:-60}"
+export HIVE_HQL_PARALLEL="${HIVE_HQL_PARALLEL:-4}"
+export START_CLEANUP_ON_FAILURE="${START_CLEANUP_ON_FAILURE:-1}"
export HIVE_MODE="${HIVE_MODE:-refresh}"
export HIVE_MODULES="${HIVE_MODULES:-all}"
HIVE_SHARED_ID="doris-shared"
@@ -172,10 +175,31 @@ fi
echo "Components are: ${COMPONENTS}"
echo "Container UID: ${CONTAINER_UID}"
echo "Stop: ${STOP}"
+echo "Start progress interval: ${START_PROGRESS_INTERVAL}"
echo "Hive mode: ${HIVE_MODE}"
echo "Hive modules: ${HIVE_MODULES}"
+echo "Hive HQL parallel: ${HIVE_HQL_PARALLEL}"
echo "Hive host alias: ${HIVE_HOST_ALIAS}"
+if ! [[ "${START_PROGRESS_INTERVAL}" =~ ^[1-9][0-9]*$ ]]; then
+ echo "Invalid start progress interval: ${START_PROGRESS_INTERVAL}"
+ usage
+fi
+
+if ! [[ "${HIVE_HQL_PARALLEL}" =~ ^[1-9][0-9]*$ ]]; then
+ echo "Invalid hive HQL parallel: ${HIVE_HQL_PARALLEL}"
+ usage
+fi
+
+case "${START_CLEANUP_ON_FAILURE}" in
+0|1)
+ ;;
+*)
+ echo "Invalid start cleanup on failure: ${START_CLEANUP_ON_FAILURE}"
+ usage
+ ;;
+esac
+
case "${HIVE_MODE}" in
fast|refresh|rebuild)
;;
@@ -598,6 +622,7 @@ declare -A START_PIDS=()
declare -A START_LOGS=()
declare -A START_COMPOSE_FILES=()
declare -A START_ENV_FILES=()
+declare -A START_DONE=()
START_ORDER=()
register_stack_metadata() {
@@ -661,9 +686,77 @@ register_job() {
START_PIDS["${component}"]="${pid}"
START_LOGS["${component}"]="${log_file}"
+ START_DONE["${component}"]=0
START_ORDER+=("${component}")
}
+cleanup_started_stacks() {
+ local component
+ local compose_file
+ local env_file
+ local i
+
+ [[ "${START_CLEANUP_ON_FAILURE}" -eq 1 ]] || return 0
+
+ for ((i = ${#START_ORDER[@]} - 1; i >= 0; --i)); do
+ component="${START_ORDER[$i]}"
+ compose_file="${START_COMPOSE_FILES["${component}"]:-}"
+ env_file="${START_ENV_FILES["${component}"]:-}"
+ [[ -n "${compose_file}" ]] || continue
+ echo "Cleaning component '${component}' after startup failure" >&2
+ compose_down_stack "${compose_file}" "${env_file}" --remove-orphans >/dev/null 2>&1 || true
+ done
+}
+
+wait_remaining_jobs_quietly() {
+ local component
+ local pid
+
+ for component in "${START_ORDER[@]}"; do
+ [[ "${START_DONE["${component}"]:-0}" -eq 0 ]] || continue
+ pid="${START_PIDS["${component}"]:-}"
+ [[ -n "${pid}" ]] || continue
+ wait "${pid}" >/dev/null 2>&1 || true
+ START_DONE["${component}"]=1
+ done
+}
+
+handle_start_failure() {
+ local component="$1"
+ local status="$2"
+
+ dump_start_failure "${component}" "${status}"
+ kill_running_jobs
+ wait_remaining_jobs_quietly
+ cleanup_started_stacks
+}
+
+collect_one_finished_job() {
+ local component
+ local pid
+ local status
+
+ for component in "${START_ORDER[@]}"; do
+ [[ "${START_DONE["${component}"]:-0}" -eq 0 ]] || continue
+ pid="${START_PIDS["${component}"]:-}"
+ [[ -n "${pid}" ]] || continue
+ if kill -0 "${pid}" >/dev/null 2>&1; then
+ continue
+ fi
+
+ status=0
+ wait "${pid}" || status=$?
+ START_DONE["${component}"]=1
+ if [[ "${status}" -ne 0 ]]; then
+ handle_start_failure "${component}" "${status}"
+ return 1
+ fi
+ return 0
+ done
+
+ return 2
+}
+
launch_component() {
local component="$1"
local log_file="$2"
@@ -679,12 +772,37 @@ kill_running_jobs() {
local pid
for component in "${START_ORDER[@]}"; do
+ [[ "${START_DONE["${component}"]:-0}" -eq 0 ]] || continue
pid="${START_PIDS["${component}"]:-}"
[[ -n "${pid}" ]] || continue
kill "${pid}" >/dev/null 2>&1 || true
done
}
+print_wait_progress() {
+ local component
+ local pid
+ local log_file
+
+ echo "Still waiting for docker components:"
+ for component in "${START_ORDER[@]}"; do
+ [[ "${START_DONE["${component}"]:-0}" -eq 0 ]] || continue
+ pid="${START_PIDS["${component}"]:-}"
+ [[ -n "${pid}" ]] || continue
+ if ! kill -0 "${pid}" >/dev/null 2>&1; then
+ continue
+ fi
+
+ log_file="${START_LOGS["${component}"]:-}"
+ echo " ${component} (pid=${pid}, log=${log_file})"
+ if [[ -n "${log_file}" && -f "${log_file}" ]]; then
+ echo " ----- ${component} log tail -----"
+ tail -n 20 "${log_file}" || true
+ echo " ----- end ${component} log tail -----"
+ fi
+ done
+}
+
dump_start_failure() {
local component="$1"
local status="$2"
@@ -742,32 +860,37 @@ print_started_summary() {
}
wait_for_started_jobs() {
- local remaining=("${START_ORDER[@]}")
- local next_remaining=()
local component
- local pid
- local status
+ local remaining_count
+ local collect_status
+ local last_progress_ts
+ local now_ts
- while (( ${#remaining[@]} > 0 )); do
- next_remaining=()
- for component in "${remaining[@]}"; do
- pid="${START_PIDS["${component}"]}"
- if kill -0 "${pid}" >/dev/null 2>&1; then
- next_remaining+=("${component}")
- continue
- fi
+ last_progress_ts="$(date +%s)"
- status=0
- wait "${pid}" || status=$?
- if [[ "${status}" -ne 0 ]]; then
- kill_running_jobs
- dump_start_failure "${component}" "${status}"
- return 1
+ while true; do
+ remaining_count=0
+ for component in "${START_ORDER[@]}"; do
+ if [[ "${START_DONE["${component}"]:-0}" -eq 0 ]]; then
+ remaining_count=$((remaining_count + 1))
fi
done
- remaining=("${next_remaining[@]}")
- if (( ${#remaining[@]} > 0 )); then
+ if (( remaining_count == 0 )); then
+ return 0
+ fi
+
+ collect_status=0
+ collect_one_finished_job || collect_status=$?
+ if [[ "${collect_status}" -eq 1 ]]; then
+ return 1
+ fi
+ if [[ "${collect_status}" -eq 2 ]]; then
+ now_ts="$(date +%s)"
+ if (( now_ts - last_progress_ts >= START_PROGRESS_INTERVAL )); then
+ print_wait_progress
+ last_progress_ts="${now_ts}"
+ fi
sleep 1
fi
done
@@ -1159,6 +1282,7 @@ exec_hive_script() {
sudo docker exec -i \
-e HIVE_BOOTSTRAP_GROUPS="${HIVE_BOOTSTRAP_GROUPS}" \
-e LOAD_PARALLEL="${LOAD_PARALLEL}" \
+ -e HIVE_HQL_PARALLEL="${HIVE_HQL_PARALLEL}" \
-e HIVE_MODULES="${HIVE_MODULES}" \
-e HIVE_BASELINE_VERSION="${HIVE_BASELINE_VERSION}" \
-e HIVE_STATE_DIR="/mnt/state" \
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org