You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2026/07/27 15:19:31 UTC
(hop) branch main updated: IT and marketplace fixes, fixes #7641 (#7642)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 5ac879b666 IT and marketplace fixes, fixes #7641 (#7642)
5ac879b666 is described below
commit 5ac879b66615219ac6182ffaa63e0e3eedb45aee
Author: Hans Van Akelyen <ha...@gmail.com>
AuthorDate: Mon Jul 27 17:19:27 2026 +0200
IT and marketplace fixes, fixes #7641 (#7642)
---
docker/integration-tests/unit-tests.Dockerfile | 40 ++++++++---
integration-tests/scripts/run-tests-docker.sh | 22 +++++--
integration-tests/scripts/run-tests.sh | 91 +++++++++++++++++++++++++-
plugins/misc/marketplace/README.md | 6 +-
plugins/misc/marketplace/pom.xml | 56 ++++++++++++----
tools/generate-full-client-env.sh | 71 --------------------
tools/install-wave1-plugins.sh | 10 ++-
7 files changed, 191 insertions(+), 105 deletions(-)
diff --git a/docker/integration-tests/unit-tests.Dockerfile b/docker/integration-tests/unit-tests.Dockerfile
index 90e58676f1..a7419c935d 100644
--- a/docker/integration-tests/unit-tests.Dockerfile
+++ b/docker/integration-tests/unit-tests.Dockerfile
@@ -61,31 +61,53 @@ RUN apt-get update \
ttf-mscorefonts-installer \
locales \
&& apt-get clean \
- && mkdir ${VOLUME_MOUNT_POINT} \
- && addgroup -gid ${JENKINS_GID} ${JENKINS_GROUP} \
- && useradd -m -d /home/${JENKINS_USER} -u ${JENKINS_UID} -g ${JENKINS_GROUP} ${JENKINS_USER} \
- && chown ${JENKINS_USER}:${JENKINS_GROUP} ${DEPLOYMENT_PATH} \
- && chown ${JENKINS_USER}:${JENKINS_GROUP} ${VOLUME_MOUNT_POINT} \
&& sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
&& locale-gen \
# Install parquet-tools from Python
&& pip3 install parquet-tools
+# Create the container identity from the host UID/GID that run-tests-docker.sh passes in, so
+# writes into the bind-mounted integration-tests/ tree land as the workspace owner.
+# Those host IDs regularly collide with distro accounts — macOS hands over staff/20, but on
+# Ubuntu GID 20 is `dialout` and its own `staff` group sits at GID 50 — so reuse whatever
+# already holds the requested ID instead of failing the build. Ownership is applied
+# numerically everywhere below for the same reason: the group *name* may resolve to a
+# different GID than the one the host asked for.
+RUN set -eu; \
+ mkdir -p ${VOLUME_MOUNT_POINT}; \
+ if ! getent group "${JENKINS_GID}" >/dev/null; then \
+ groupadd -g "${JENKINS_GID}" "${JENKINS_GROUP}" 2>/dev/null \
+ || groupadd -g "${JENKINS_GID}" "hopgrp${JENKINS_GID}"; \
+ fi; \
+ if getent passwd "${JENKINS_UID}" >/dev/null; then \
+ existing_user="$(getent passwd "${JENKINS_UID}" | cut -d: -f1)"; \
+ if [ "${existing_user}" != "${JENKINS_USER}" ]; then \
+ usermod -l "${JENKINS_USER}" "${existing_user}"; \
+ fi; \
+ usermod -d "/home/${JENKINS_USER}" -g "${JENKINS_GID}" "${JENKINS_USER}"; \
+ elif getent passwd "${JENKINS_USER}" >/dev/null; then \
+ usermod -u "${JENKINS_UID}" -g "${JENKINS_GID}" -d "/home/${JENKINS_USER}" "${JENKINS_USER}"; \
+ else \
+ useradd -m -d "/home/${JENKINS_USER}" -u "${JENKINS_UID}" -g "${JENKINS_GID}" "${JENKINS_USER}"; \
+ fi; \
+ mkdir -p "/home/${JENKINS_USER}"; \
+ chown ${JENKINS_UID}:${JENKINS_GID} "/home/${JENKINS_USER}" ${DEPLOYMENT_PATH} ${VOLUME_MOUNT_POINT}
-COPY --chown=${JENKINS_USER}:${JENKINS_GROUP} ./assemblies/client/target/hop ${DEPLOYMENT_PATH}/hop
+
+COPY --chown=${JENKINS_UID}:${JENKINS_GID} ./assemblies/client/target/hop ${DEPLOYMENT_PATH}/hop
# Wave 1 optional plugins (marketplace) — expect host to have run tools/install-wave1-plugins.sh
# into assemblies/client/target/hop before this build (run-tests-docker.sh and Jenkins do this).
# If a plugin is still missing at runtime, corresponding ITs will fail clearly.
# Copy gcp key
-COPY --chown=${JENKINS_USER}:${JENKINS_GROUP} ${GCP_KEY_FILE} /tmp/google-key-apache-hop-it.json
+COPY --chown=${JENKINS_UID}:${JENKINS_GID} ${GCP_KEY_FILE} /tmp/google-key-apache-hop-it.json
# Copy mail keystore
-COPY --chown=${JENKINS_USER}:${JENKINS_GROUP} ./docker/integration-tests/resource/mail/conf/keystore /tmp
+COPY --chown=${JENKINS_UID}:${JENKINS_GID} ./docker/integration-tests/resource/mail/conf/keystore /tmp
# Unzip and install in correct location
-RUN chown -R ${JENKINS_USER}:${JENKINS_GROUP} ${DEPLOYMENT_PATH}/hop \
+RUN chown -R ${JENKINS_UID}:${JENKINS_GID} ${DEPLOYMENT_PATH}/hop \
&& chmod 700 ${DEPLOYMENT_PATH}/hop/*.sh
# make volume available so that hop pipeline and workflow files can be provided easily
diff --git a/integration-tests/scripts/run-tests-docker.sh b/integration-tests/scripts/run-tests-docker.sh
index 8d70ef94bc..643aed0e7c 100755
--- a/integration-tests/scripts/run-tests-docker.sh
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -110,20 +110,30 @@ if [ -z "${GCP_KEY_FILE}" ]; then
GCP_KEY_FILE="./docker/integration-tests/resource/dummyfile"
fi
+GCP_KEY_PATH="${GCP_KEY_FILE}"
+case "${GCP_KEY_PATH}" in
+/*) ;;
+*)
+ if [ -f "$(cd "${CURRENT_DIR}/../.." && pwd)/${GCP_KEY_PATH#./}" ]; then
+ GCP_KEY_PATH="$(cd "${CURRENT_DIR}/../.." && pwd)/${GCP_KEY_PATH#./}"
+ fi
+ ;;
+esac
+
# Detect a real Google Cloud service-account key. The dummy file is a license comment, not
# JSON; spreadsheet Google Sheets ITs need a real key (Jenkins: credentials gcp-access-hop).
# Require non-empty file + type=service_account + JSON-looking content so a corrupt/empty
# secret still skips cleanly. When python3 is available, also require parseable JSON.
SKIP_GOOGLE_SHEETS="false"
GCP_KEY_OK="true"
-if [ ! -f "${GCP_KEY_FILE}" ] \
- || [[ "${GCP_KEY_FILE}" == *dummyfile* ]] \
- || [ ! -s "${GCP_KEY_FILE}" ] \
- || ! grep -qE '"type"[[:space:]]*:[[:space:]]*"service_account"' "${GCP_KEY_FILE}" 2>/dev/null \
- || ! grep -qE '\{' "${GCP_KEY_FILE}" 2>/dev/null; then
+if [ ! -f "${GCP_KEY_PATH}" ] \
+ || [[ "${GCP_KEY_PATH}" == *dummyfile* ]] \
+ || [ ! -s "${GCP_KEY_PATH}" ] \
+ || ! grep -qE '"type"[[:space:]]*:[[:space:]]*"service_account"' "${GCP_KEY_PATH}" 2>/dev/null \
+ || ! grep -qE '\{' "${GCP_KEY_PATH}" 2>/dev/null; then
GCP_KEY_OK="false"
elif command -v python3 >/dev/null 2>&1; then
- if ! python3 -c "import json,sys; json.load(open(sys.argv[1]))" "${GCP_KEY_FILE}" 2>/dev/null; then
+ if ! python3 -c "import json,sys; json.load(open(sys.argv[1]))" "${GCP_KEY_PATH}" 2>/dev/null; then
GCP_KEY_OK="false"
fi
fi
diff --git a/integration-tests/scripts/run-tests.sh b/integration-tests/scripts/run-tests.sh
index de7da572ba..6116c21845 100755
--- a/integration-tests/scripts/run-tests.sh
+++ b/integration-tests/scripts/run-tests.sh
@@ -190,6 +190,35 @@ should_run_workflow() {
return 1
}
+# Emit one "STATUS<TAB>NAME<TAB>TIME" line per <testcase> in a surefire XML report.
+# The single-JVM suite runner records each test's outcome in that report rather than in this
+# script's loop, so it is the only place the per-test breakdown still exists.
+# Parsed with python3 (installed in the IT image) rather than grep/awk: every testcase embeds
+# its full workflow log in a CDATA section, and that log text can contain anything a line-based
+# parser would mistake for markup.
+parse_surefire_testcases() {
+ python3 - "$1" <<'PYTHON_PARSE_SUREFIRE' 2>/dev/null
+import sys
+import xml.etree.ElementTree as ET
+
+try:
+ root = ET.parse(sys.argv[1]).getroot()
+except Exception:
+ sys.exit(1)
+
+for testcase in root.iter("testcase"):
+ if testcase.find("failure") is not None:
+ status = "FAIL"
+ elif testcase.find("error") is not None:
+ status = "ERROR"
+ elif testcase.find("skipped") is not None:
+ status = "SKIP"
+ else:
+ status = "PASS"
+ print("%s\t%s\t%s" % (status, testcase.get("name", ""), testcase.get("time", "")))
+PYTHON_PARSE_SUREFIRE
+}
+
# Set up a temporary folder
export TMP_FOLDER=/tmp/hop-it-$$
rm -rf "${TMP_FOLDER}"
@@ -312,9 +341,6 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
if (($exit_code >= 1)); then
errors_counter=1
failures_counter=1
- echo "${PROJECT_NAME}" >>"${CURRENT_DIR}"/../surefire-reports/failed_tests
- else
- echo "${PROJECT_NAME}" >>"${CURRENT_DIR}"/../surefire-reports/passed_tests
fi
echo ${SPACER}
@@ -337,6 +363,65 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
fi
fi
+ # Every main*.hwf ran inside one JVM, so this loop never saw the individual tests.
+ # Replay the per-test outcomes from the surefire report the suite just wrote, so the
+ # console keeps its per-test breakdown and the passed_tests/failed_tests overview files
+ # (printed at the end of run-tests-docker.sh) list test names rather than one project
+ # name. Runs after the fallback report above on purpose: a suite that died on startup
+ # then shows up here as a failed "suite_startup" test instead of vanishing.
+ SUITE_RESULTS="${TMP_FOLDER}/suite-results-${PROJECT_NAME}.tsv"
+ : >"${SUITE_RESULTS}"
+ if [ -f "${SUREFIRE_DIR}/surefile_${PROJECT_NAME}.xml" ]; then
+ parse_surefire_testcases "${SUREFIRE_DIR}/surefile_${PROJECT_NAME}.xml" \
+ >"${SUITE_RESULTS}" || : >"${SUITE_RESULTS}"
+ fi
+
+ if [ -s "${SUITE_RESULTS}" ]; then
+ suite_passed=0
+ suite_failed=0
+ suite_skipped=0
+
+ echo ${SPACER}
+ echo "Test results: ${PROJECT_NAME}"
+ echo ${SPACER}
+
+ while IFS=$'\t' read -r tc_status tc_name tc_time; do
+ [ -z "${tc_name}" ] && continue
+ case "${tc_status}" in
+ PASS)
+ suite_passed=$((suite_passed + 1))
+ echo -e "\033[1;32mPASSED \033[0m ${tc_name} (${tc_time}s)"
+ echo "${tc_name}" >>"${CURRENT_DIR}"/../surefire-reports/passed_tests
+ ;;
+ SKIP)
+ suite_skipped=$((suite_skipped + 1))
+ echo -e "\033[1;93mSKIPPED\033[0m ${tc_name}"
+ ;;
+ *)
+ suite_failed=$((suite_failed + 1))
+ echo -e "\033[1;91mFAILED \033[0m ${tc_name} (${tc_time}s)"
+ echo "${tc_name}" >>"${CURRENT_DIR}"/../surefire-reports/failed_tests
+ ;;
+ esac
+ done <"${SUITE_RESULTS}"
+
+ echo ${SPACER}
+ echo "${PROJECT_NAME}: ${suite_passed} passed, ${suite_failed} failed, ${suite_skipped} skipped"
+
+ # A non-zero hop-run exit that no testcase accounts for (e.g. the suite aborted after
+ # the report was written) must still surface as a failure rather than an all-green list.
+ if (($exit_code >= 1)) && ((suite_failed == 0)); then
+ echo "${PROJECT_NAME} (suite exited ${exit_code})" >>"${CURRENT_DIR}"/../surefire-reports/failed_tests
+ fi
+ else
+ # No parseable report at all: fall back to a single project-level entry.
+ if (($exit_code >= 1)); then
+ echo "${PROJECT_NAME}" >>"${CURRENT_DIR}"/../surefire-reports/failed_tests
+ else
+ echo "${PROJECT_NAME}" >>"${CURRENT_DIR}"/../surefire-reports/passed_tests
+ fi
+ fi
+
else
# Classic path: one hop-run JVM per main-*.hwf
diff --git a/plugins/misc/marketplace/README.md b/plugins/misc/marketplace/README.md
index 101148c6fe..4b09e2b425 100644
--- a/plugins/misc/marketplace/README.md
+++ b/plugins/misc/marketplace/README.md
@@ -232,7 +232,7 @@ Do not hardcode plugin lists elsewhere. Everything below reads this registry:
| Consumer | How it uses the registry |
|----------|--------------------------|
| GUI catalog / `hop marketplace query` | Loads YAML at runtime |
-| `full-client-env.yaml` | Generated by `tools/generate-full-client-env.sh` (Maven `generate-resources`) |
+| `full-client-env.yaml` | Generated from this registry during the marketplace build (Maven `generate-resources`) |
| Local publish to Nexus/Artifactory | `docker/marketplace-nexus/publish-marketplace-plugins.sh` |
| Unpack zips into an install (IT images) | `tools/install-wave1-plugins.sh` via `tools/list-marketplace-plugins.sh` |
@@ -355,8 +355,8 @@ publish side (admin / CI user).
# Unzip all built registry zips into a Hop install (no Maven repo)
./tools/install-wave1-plugins.sh [HOP_INSTALL_DIR]
-# Regenerate full-client-env.yaml (also run by marketplace module generate-resources)
-./tools/generate-full-client-env.sh [HOP_VERSION] [OUTPUT_PATH]
+# Regenerate full-client-env.yaml (normally produced automatically by the marketplace build)
+./mvnw -o -pl plugins/misc/marketplace generate-resources
```
## Packaging & size
diff --git a/plugins/misc/marketplace/pom.xml b/plugins/misc/marketplace/pom.xml
index f9aaf689aa..2544668319 100644
--- a/plugins/misc/marketplace/pom.xml
+++ b/plugins/misc/marketplace/pom.xml
@@ -51,25 +51,59 @@
<build>
<plugins>
- <!-- Generate full-client-env.yaml from optional-plugins.yaml (shipped at install root). -->
+ <!--
+ Generate full-client-env.yaml from optional-plugins.yaml (the single source of
+ truth) and ship it at the install root (see src/assembly/assembly.xml).
+ Uses maven-antrun-plugin (pure Java, no external shell) so the build stays
+ platform-agnostic. NOTE: the <echo> body below is intentionally flush-left —
+ Ant preserves leading whitespace verbatim, so do not indent those lines.
+ -->
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>3.5.0</version>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>3.1.0</version>
<executions>
<execution>
<id>generate-full-client-env</id>
<goals>
- <goal>exec</goal>
+ <goal>run</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
- <executable>bash</executable>
- <arguments>
- <argument>${maven.multiModuleProjectDirectory}/tools/generate-full-client-env.sh</argument>
- <argument>${project.version}</argument>
- <argument>${project.build.directory}/generated/full-client-env.yaml</argument>
- </arguments>
+ <target>
+ <mkdir dir="${project.build.directory}/generated"></mkdir>
+ <!-- Keep only the '- artifactId:' entries, normalized to ' - artifactId: <id>'. -->
+ <loadfile property="fullClientEnv.plugins" srcFile="${project.basedir}/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml">
+ <filterchain>
+ <linecontainsregexp>
+ <regexp pattern="^\s*-\s*artifactId:\s*\S"></regexp>
+ </linecontainsregexp>
+ <tokenfilter>
+ <replaceregex pattern="^\s*-\s*artifactId:\s*(.*?)\s*$" replace=" - artifactId: \1"></replaceregex>
+ </tokenfilter>
+ </filterchain>
+ </loadfile>
+ <fail message="No '- artifactId:' entries found in optional-plugins.yaml" unless="fullClientEnv.plugins"></fail>
+ <echo encoding="UTF-8" file="${project.build.directory}/generated/full-client-env.yaml"># Generated from optional-plugins.yaml - do not edit by hand.
+# Rebuild the marketplace module to refresh (Maven generate-resources).
+#
+# Restores marketplace-optional plugins that used to ship in the fat hop-client.
+# From the Hop install directory (network required for ASF/Central or your mirror):
+#
+# ./hop marketplace apply -f full-client-env.yaml
+#
+# Then restart Hop so the plugin registry reloads.
+# Do not use --prune unless you intend to remove other marketplace installs.
+#
+version: "1.0"
+hopVersion: "${project.version}"
+enforceOnRun: false
+# repositories omitted: uses hop-config defaults (ASF primary + Maven Central)
+plugins:
+${fullClientEnv.plugins}</echo>
+ <!-- Normalize line endings so the output is byte-identical on every platform. -->
+ <fixcrlf encoding="UTF-8" eol="lf" file="${project.build.directory}/generated/full-client-env.yaml"></fixcrlf>
+ </target>
</configuration>
</execution>
</executions>
diff --git a/tools/generate-full-client-env.sh b/tools/generate-full-client-env.sh
deleted file mode 100755
index 36d97d86e9..0000000000
--- a/tools/generate-full-client-env.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Generate full-client-env.yaml from optional-plugins.yaml (marketplace registry).
-#
-# Usage:
-# ./tools/generate-full-client-env.sh [HOP_VERSION] [OUTPUT_PATH]
-#
-# Defaults:
-# HOP_VERSION from ${project.version} when set by Maven, else 2.19.0-SNAPSHOT
-# OUTPUT_PATH plugins/misc/marketplace/target/generated/full-client-env.yaml
-#
-set -euo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-REGISTRY="${ROOT}/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml"
-VERSION="${1:-${HOP_VERSION:-2.19.0-SNAPSHOT}}"
-OUT="${2:-${ROOT}/plugins/misc/marketplace/target/generated/full-client-env.yaml}"
-
-if [[ ! -f "${REGISTRY}" ]]; then
- echo "Registry not found: ${REGISTRY}" >&2
- exit 1
-fi
-
-mkdir -p "$(dirname "${OUT}")"
-
-# Extract artifactIds (lines like " - artifactId: hop-tech-parquet").
-# Use a read loop instead of `mapfile` (bash 4+) so this also runs on
-# older bash configurations
-ARTIFACTS=()
-while IFS= read -r artifact; do
- ARTIFACTS+=("${artifact}")
-done < <(sed -n 's/^[[:space:]]*- artifactId:[[:space:]]*\(.*\)$/\1/p' "${REGISTRY}")
-
-if [[ ${#ARTIFACTS[@]} -eq 0 ]]; then
- echo "No plugins found in ${REGISTRY}" >&2
- exit 1
-fi
-
-{
- cat <<EOF
-# Generated from optional-plugins.yaml — do not edit by hand.
-# Rebuild marketplace / hop-client to refresh (tools/generate-full-client-env.sh).
-#
-# Restores marketplace-optional plugins that used to ship in the fat hop-client.
-# From the Hop install directory (network required for ASF/Central or your mirror):
-#
-# ./hop marketplace apply -f full-client-env.yaml
-#
-# Then restart Hop so the plugin registry reloads.
-# Do not use --prune unless you intend to remove other marketplace installs.
-#
-version: "1.0"
-hopVersion: "${VERSION}"
-enforceOnRun: false
-# repositories omitted: uses hop-config defaults (ASF primary + Maven Central)
-plugins:
-EOF
- for id in "${ARTIFACTS[@]}"; do
- echo " - artifactId: ${id}"
- done
-} >"${OUT}"
-
-echo "Wrote ${OUT} (${#ARTIFACTS[@]} plugins, hopVersion=${VERSION})"
diff --git a/tools/install-wave1-plugins.sh b/tools/install-wave1-plugins.sh
index 7a81cacc55..71994e756b 100755
--- a/tools/install-wave1-plugins.sh
+++ b/tools/install-wave1-plugins.sh
@@ -34,11 +34,17 @@ if [[ ! -x "${LIST_SCRIPT}" ]]; then
chmod +x "${LIST_SCRIPT}" 2>/dev/null || true
fi
-mapfile -t PLUGINS < <(HOP_VERSION="${VERSION}" "${LIST_SCRIPT}" --zips)
+# `mapfile` is a bash 4+ builtin and macOS still ships bash 3.2, where it aborts the whole
+# script under `set -e` and no plugin ever gets installed. Read the list portably instead.
+PLUGINS=()
+while IFS= read -r entry; do
+ [[ -n "${entry}" ]] && PLUGINS+=("${entry}")
+done < <(HOP_VERSION="${VERSION}" "${LIST_SCRIPT}" --zips)
installed=0
skipped=0
-for entry in "${PLUGINS[@]}"; do
+# bash 3.2 treats an empty array as unbound under `set -u`; expand defensively.
+for entry in ${PLUGINS[@]+"${PLUGINS[@]}"}; do
id="${entry%%|*}"
rel="${entry#*|}"
zip="${ROOT}/${rel}"