You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ta...@apache.org on 2026/09/29 23:49:01 UTC
(hbase) branch HBASE-30087 updated: HBASE-30437: Run Read-Replica Integration tests within a test environment container (#8714)
This is an automated email from the ASF dual-hosted git repository.
taklwu pushed a commit to branch HBASE-30087
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/HBASE-30087 by this push:
new 798f474a798 HBASE-30437: Run Read-Replica Integration tests within a test environment container (#8714)
798f474a798 is described below
commit 798f474a79829dde74b781951d09bf15b734c6e7
Author: Kevin Geiszler <ke...@gmail.com>
AuthorDate: Tue Sep 29 19:48:54 2026 -0400
HBASE-30437: Run Read-Replica Integration tests within a test environment container (#8714)
Signed-off-by: Tak Lon (Stephen) Wu <ta...@apache.org>
---
dev-support/docker/Dockerfile | 4 +
dev-support/read-replica/.env | 2 +-
dev-support/read-replica/README.md | 257 ++++++++++++--------
.../hbase_nightly_read_replica_test.sh | 265 +++++++++++++--------
...st.sh => run_read_replica_integration_tests.sh} | 101 +++++++-
5 files changed, 433 insertions(+), 196 deletions(-)
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index cc587b829cc..717c4c06705 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -58,6 +58,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
libxml2-dev='2.9.13+dfsg-*' \
libxml2-utils='2.9.13+dfsg-*' \
zip='3.0-*' \
+# read-replica nightly tests use this Dockerfile and need these docker and python dependencies
+ docker.io='20.10.*' \
+ docker-compose-v2='2.*' \
+ python3-venv='3.10.6-*' \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/* \
diff --git a/dev-support/read-replica/.env b/dev-support/read-replica/.env
index 1b36c57226b..e041467192c 100644
--- a/dev-support/read-replica/.env
+++ b/dev-support/read-replica/.env
@@ -35,4 +35,4 @@ DOCKER_COMPOSE_FILE=${HBASE_ROOT}/dev-support/read-replica/docker-compose.yml
# The location of the utils directory within the docker container
CONTAINER_UTILS_DIR=/opt/utils
# The log level for the Python scripts
-LOG_LEVEL=DEBUG
+LOG_LEVEL=INFO
diff --git a/dev-support/read-replica/README.md b/dev-support/read-replica/README.md
index 0b6821021c2..a760c1e1532 100644
--- a/dev-support/read-replica/README.md
+++ b/dev-support/read-replica/README.md
@@ -23,47 +23,62 @@ Integration test framework for HBase's Read-Replica feature, driven by Jenkins a
## Why Docker?
HBase's `MiniHBaseCluster` cannot support multi-cluster Read-Replica testing because the
-`META_TABLE_NAME` static variable is shared across JVMs (see HBASE-29691). This framework
+static `META_TABLE_NAME` variable is shared across JVMs (see HBASE-29691). This framework
sidesteps that limitation by running two fully isolated HBase clusters in Docker containers
that share a filesystem-based `hbase.rootdir`.
## Architecture
```
-┌─────────────────────────────────────────────────────────┐
-│ Jenkins Agent │
-│ │
-│ ┌──────────────────┐ ┌──────────────────┐ │
-│ │ hbase-docker │ │ hbase-docker-2 │ │
-│ │ (Active Cluster)│ │ (Replica Cluster)│ │
-│ │ │ │ │ │
-│ │ ZooKeeper │ │ ZooKeeper │ │
-│ │ HMaster │ │ HMaster │ │
-│ │ RegionServer │ │ RegionServer │ │
-│ │ │ │ │ │
-│ │ read-only=false │ │ read-only=true │ │
-│ │ suffix=(empty) │ │ suffix=replica1 │ │
-│ └────────┬─────────┘ └────────┬─────────┘ │
-│ │ │ │
-│ └────────────┬───────────────┘ │
-│ │ │
-│ ┌─────────▼───────────┐ │
-│ │ Shared Data Store │ │
-│ │ (hbase.rootdir) │ │
-│ │ │ │
-│ │ /data-store/hbase │ │
-│ └─────────────────────┘ │
-│ │
-│ ┌──────────────────────────────────────────────────┐ │
-│ │ Python Test Scripts │ │
-│ │ (communicate via `docker exec` + HBase Shell) │ │
-│ └──────────────────────────────────────────────────┘ │
-└─────────────────────────────────────────────────────────┘
+┌──────────────────────────────────────────────────────────────────────┐
+│ Host (Jenkins Agent or Local) │
+│ │
+│ ┌────────────────────────────────────────────────────────────────┐ │
+│ │ Test Environment Container (dev-support image) │ │
+│ │ Built from: dev-support/docker/Dockerfile │ │
+│ │ Contains: JDK, Maven, Python, Docker CLI │ │
+│ │ Entry: run_read_replica_integration_tests.sh │ │
+│ │ │ │
+│ │ ┌──────────────────┐ ┌──────────────────┐ │ │
+│ │ │ hbase-docker │ │ hbase-docker-2 │ │ │
+│ │ │ (Active Cluster)│ │ (Replica Cluster)│ │ │
+│ │ │ │ │ │ │ │
+│ │ │ ZooKeeper │ │ ZooKeeper │ │ │
+│ │ │ HMaster │ │ HMaster │ │ │
+│ │ │ RegionServer │ │ RegionServer │ │ │
+│ │ │ │ │ │ │ │
+│ │ │ read-only=false │ │ read-only=true │ │ │
+│ │ │ suffix=(empty) │ │ suffix=replica1 │ │ │
+│ │ └────────┬─────────┘ └────────┬─────────┘ │ │
+│ │ │ │ │ │
+│ │ └────────────┬───────────────┘ │ │
+│ │ │ │ │
+│ │ ┌─────────▼───────────┐ │ │
+│ │ │ Shared Data Store │ │ │
+│ │ │ (hbase.rootdir) │ │ │
+│ │ │ │ │ │
+│ │ │ /data-store/hbase │ │ │
+│ │ └─────────────────────┘ │ │
+│ │ │ │
+│ │ ┌──────────────────────────────────────────────────────┐ │ │
+│ │ │ Python Test Scripts │ │ │
+│ │ │ (communicate via `docker exec` + HBase Shell) │ │ │
+│ │ └──────────────────────────────────────────────────────┘ │ │
+│ └────────────────────────────────────────────────────────────────┘ │
+│ │
+│ Docker socket bind-mounted into the test environment container │
+│ (Docker-outside-of-Docker / DooD) │
+└──────────────────────────────────────────────────────────────────────┘
```
-Both clusters mount the same `data-store/hbase` directory as their `hbase.rootdir`. The active
-cluster writes data and the replica cluster reads it after explicit `refresh_meta` /
-`refresh_hfiles` calls. Each cluster has its own ZooKeeper instance and distinct
+The test environment container is built from `dev-support/docker/Dockerfile` and provides
+JDK, Maven, Python, and the Docker CLI. It uses Docker-outside-of-Docker (DooD) by
+bind-mounting the host's Docker socket, so the HBase cluster containers run as siblings on
+the host's Docker daemon.
+
+Both HBase clusters mount the same `data-store/hbase` directory as their `hbase.rootdir`.
+The active cluster writes data and the replica cluster reads it after explicit `refresh_meta`
+/ `refresh_hfiles` calls. Each cluster has its own ZooKeeper instance and distinct
`hbase.meta.table.suffix` to avoid meta table collisions.
## Directory Structure
@@ -75,6 +90,8 @@ read-replica/
├── build-images.sh # Builds the Docker image from the local HBase checkout
├── docker-compose.yml # Defines the two container services
├── requirements.txt # Python dependencies
+├── hbase_nightly_read_replica_test.sh # Outer driver script (runs on host)
+├── run_read_replica_integration_tests.sh # Inner test runner (runs inside container)
├── cluster1/ # Active cluster configuration
│ └── conf/
│ ├── hbase-site.xml # hbase.global.readonly.enabled=false
@@ -88,7 +105,7 @@ read-replica/
├── python/
│ ├── proto/
│ │ ├── ActiveClusterSuffix.proto # Automatically copied to this location by
-│ │ │ # hbase_nightly_read_replica_test.sh
+│ │ │ # run_read_replica_integration_tests.sh
│ │ └── proto_compiler.py # Compiles .proto files into python/proto/generated/
│ ├── scripts/
│ │ └── verify_hbase_start.py # Standalone startup verification (not part of pytest suite)
@@ -110,32 +127,65 @@ read-replica/
└── tsv_generator.py # Generates random TSV data for bulkloading
```
+## Scripts
+
+The test framework uses two scripts in a layered architecture:
+
+### `hbase_nightly_read_replica_test.sh` (Outer Driver)
+
+Runs on the host (Jenkins agent or local machine). It builds a test environment Docker image
+from `dev-support/docker/Dockerfile`, starts a container, and either runs the inner test
+script automatically or drops into an interactive shell for development.
+
+| Step | Description |
+|------|-------------|
+| 1 | Build the test environment Docker image (`hbase-dev-support:<build-number>`) |
+| 2 | Start a container from that image, mounting the HBase repo, Docker socket, and `.m2` cache |
+| 3 | Run `run_read_replica_integration_tests.sh` inside the container (or start an interactive shell in dev mode) |
+
+**Options:**
+
+| Flag | Description |
+|------|-------------|
+| `-d` / `--dev` | Start a detached dev container for interactive exploration instead of running the test suite. See [Dev Mode](#dev-mode). |
+| `-k <expression>` | Pytest `-k` filter expression forwarded to the inner test script for test selection. See [Running Specific Tests](#running-specific-tests). |
+| `-m` / `--m2 <path>` | Parent directory of the `.m2` Maven cache to bind-mount into the container. Defaults to `$HOME`. The directory `<path>/.m2` will be created if it does not exist. |
+| `-j` / `--java-version <ver>` | JVM version forwarded to the inner test script, which uses it to set `JAVA_HOME` inside the container. |
+
+### `run_read_replica_integration_tests.sh` (Inner Test Runner)
+
+Runs inside the test environment container. Normally invoked by the outer driver, but can
+also be run directly when using dev mode.
+
+| Step | Description |
+|------|-------------|
+| 1 | Clone the HBase source tree for the Docker build context (`git clone --local`) |
+| 2 | Copy and compile the `ActiveClusterSuffix.proto` protobuf definition |
+| 3 | Create a Python virtual environment and install dependencies |
+| 4 | Build the `hbase-read-replica` Docker image for the active and replica clusters |
+| 5 | Run the pytest integration suite |
+
+**Options:**
+
+| Flag | Description |
+|------|-------------|
+| `-i` / `--keep-image` | Do not remove the `hbase-read-replica` Docker image on exit. Useful for re-running tests without rebuilding. |
+| `-c` / `--keep-containers` | Do not run `docker compose down` on exit. Lets you inspect container state after tests. |
+| `-k <expression>` | Pytest `-k` filter expression for test selection. |
+| `-j` / `--java-version <ver>` | JVM version to use (default: 17). Sets `JAVA_HOME` to `/usr/lib/jvm/java-<ver>`. |
+
## CI: Jenkins Nightly Pipeline
**Files:**
- `dev-support/read-replica/Jenkinsfile` — pipeline definition (`hbase read-replica feature checks`)
-- `dev-support/read-replica/hbase_nightly_read_replica_test.sh` — test driver script
+- `dev-support/read-replica/hbase_nightly_read_replica_test.sh` — outer driver script
+- `dev-support/read-replica/run_read_replica_integration_tests.sh` — inner test runner
### When It Runs
The read-replica tests run as their own standalone nightly pipeline on the `master` and `branch-3`
branches, separate from the main HBase nightly build.
-### What the Test Driver Does
-
-`hbase_nightly_read_replica_test.sh` is invoked by the Jenkins stage and performs these steps:
-
-| # | Step | Description |
-|---|------|-------------|
-| 1 | Clone HBase source | `git clone --local` into `read-replica/hbase/` for the Docker build context (Docker COPY can't follow symlinks) |
-| 2 | Source `.env` | Loads environment variables |
-| 3 | Register cleanup trap | On exit: runs `docker compose down` (unless `--keep-containers`), removes the Docker image (unless `--keep-image`), and deletes the cloned source |
-| 4 | Copy Protobuf | Copies the latest `ActiveClusterSuffix.proto` from the source tree into `python/proto/` |
-| 5 | Set up Python environment | Creates a venv, installs dependencies from `requirements.txt` |
-| 6 | Compile Protobuf | Runs `python/proto/proto_compiler.py` |
-| 7 | Build Docker image | Runs `build-images.sh` (Maven build + Docker multi-stage build) |
-| 8 | Run test suite | Runs `pytest` on `python/test/test_read_replica_feature.py`, producing an HTML report and JUnit XML results (see [Test Suite](#test-suite) below) |
-
### On Failure
If any test fails, the build is marked `UNSTABLE` (not `FAILURE`) so that later result
@@ -158,12 +208,6 @@ so any failing test is automatically retried up to 2 times before being marked a
| `test_cannot_promote_second_active_cluster` | Disabling read-only on the replica raises `ReadOnlyTransitionException` while an active cluster exists |
| `test_bulkloaded_data_and_region_splits` | `ImportTsv` + `completebulkload` works on active, is rejected on replica; region splits propagate |
-To run a specific test:
-
-```bash
-pytest python/test/test_read_replica_feature.py::TestReadReplica::test_create_drop_behavior
-```
-
### Test Results
Pytest produces two output files in the `output/` directory:
@@ -198,67 +242,92 @@ Defines environment variables consumed by Docker Compose, the build script, and
The `ActiveClusterSuffix.proto` message defines the format of the `active.cluster.suffix.id`
file written to the shared data store. Tests compile this proto and deserialize the file to
verify that the recorded active cluster matches the expected configuration after role swaps.
+The `ActiveClusterSuffix.proto` file is copied directly from the HBase repo during runtime
+of `run_read_replica_integration_tests.sh`.
## Running Locally
-The easiest way to run the tests locally is with `hbase_nightly_read_replica_test.sh`, which
-handles environment setup, Docker image builds, and pytest execution automatically. This is also
-useful for reproducing test failures seen in CI.
+The same script used by Jenkins works locally. The only prerequisite is Docker — the test
+environment container provides JDK, Maven, Python, and everything else needed to build and
+test.
+
+### Running the Full Suite
+
+From the repo root:
```bash
-# From the repo root — run the full suite
-dev-support/read-replica/hbase_nightly_read_replica_test.sh
+bash dev-support/read-replica/hbase_nightly_read_replica_test.sh
```
-The script accepts two flags for local debugging:
+When run locally (no `BUILD_NUMBER` environment variable), the test environment image is
+preserved so subsequent runs skip the image build. In Jenkins, the image is cleaned up
+automatically.
-| Flag | Description |
-|------|-------------|
-| `-i` / `--keep-image` | Preserves the Docker image after tests finish. This lets you spin up fresh containers with `docker compose up` from the `dev-support/read-replica/` directory without rebuilding the image. |
-| `-c` / `--keep-containers` | Preserves the running containers after tests finish. This lets you `docker exec` into the containers to inspect state, logs, and HBase Shell. |
+### Running Specific Tests
+
+Use `-k` to pass a pytest `-k` filter expression that selects tests by name
+(see [pytest docs](https://docs.pytest.org/en/stable/example/markers.html#using-k-expr-to-select-tests-based-on-their-name) for details):
+
+```bash
+# Run two specific tests
+bash dev-support/read-replica/hbase_nightly_read_replica_test.sh \
+ -k "test_create_drop_behavior or test_put_get_delete_behavior"
+
+# Run a single test
+bash dev-support/read-replica/hbase_nightly_read_replica_test.sh \
+ -k "test_put_get_delete_behavior"
+```
+
+### Dev Mode
+
+Dev mode (`-d` / `--dev`) builds and starts the test environment container but does **not**
+run the test suite. This gives you an interactive environment where you can run the inner
+test script as many times as you need — useful for reproducing bugs and iterating on test
+development.
+
+**1. Start the dev container:**
```bash
-# Keep the image and containers for debugging
-dev-support/read-replica/hbase_nightly_read_replica_test.sh --keep-image --keep-containers
+bash dev-support/read-replica/hbase_nightly_read_replica_test.sh -d
```
-### Running Manually
+The script prints the container ID and instructions for entering it.
-If you prefer to run the setup steps yourself (e.g. to skip the Docker image rebuild after the
-first run):
+**2. Enter the container:**
```bash
-cd dev-support/read-replica
+docker exec -it <container-id> bash
+```
-# Override CI-specific paths
-export HBASE_ROOT="$(cd ../.. && pwd)"
+**3. Run the tests inside the container:**
-# Load remaining variables
-set -a && source .env && set +a
+```bash
+# Run the full suite
+bash run_read_replica_integration_tests.sh
-# Set Python path
-export PYTHONPATH="$(pwd)"
+# Run specific tests
+bash run_read_replica_integration_tests.sh \
+ -k "test_create_drop_behavior or test_put_get_delete_behavior"
+```
-# Install Python dependencies
-python3 -m venv .venv
-source .venv/bin/activate
-pip install -r requirements.txt
+**4. Speed up re-runs with `--keep-image`:**
-# Compile protobuf
-cp "${HBASE_ROOT}/hbase-protocol-shaded/src/main/protobuf/server/ActiveClusterSuffix.proto" python/proto/
-python3 python/proto/proto_compiler.py
+By default, the inner script removes the `hbase-read-replica` cluster Docker image on exit.
+Pass `--keep-image` to preserve it, so subsequent runs skip the Maven build and image
+creation — saving significant time during development:
-# Build Docker image (requires Maven + Docker — skip if reusing a previous image)
-./build-images.sh
+```bash
+bash run_read_replica_integration_tests.sh --keep-image
-# Run the full test suite
-pytest python/test/test_read_replica_feature.py
+# Combine with -k to iterate on a specific test
+bash run_read_replica_integration_tests.sh --keep-image \
+ -k "test_create_drop_behavior"
+```
-# Or run a single test
-pytest python/test/test_read_replica_feature.py::TestReadReplica::test_create_drop_behavior
+**5. Stop and remove the dev container when done:**
-# Clean up
-docker compose -f docker-compose.yml down
+```bash
+docker stop <container-id> && docker rm <container-id>
```
### Mounted Volumes
@@ -271,7 +340,7 @@ a clean state:
rm -rf tmp-read-replica-data
```
-**Prerequisites:** Docker, Docker Compose, Python 3, Maven, JDK 17.
+**Prerequisites:** Docker.
## Related
diff --git a/dev-support/read-replica/hbase_nightly_read_replica_test.sh b/dev-support/read-replica/hbase_nightly_read_replica_test.sh
index b6bec403ced..b2b8f36a7ae 100755
--- a/dev-support/read-replica/hbase_nightly_read_replica_test.sh
+++ b/dev-support/read-replica/hbase_nightly_read_replica_test.sh
@@ -15,119 +15,198 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-# Run the read-replica Docker integration test suite.
+# hbase_nightly_read_replica_test.sh
+#
+# Outer driver script for the HBase read-replica integration test suite.
+# Builds a dev-support Docker container image (from
+# hbase/dev-support/docker/Dockerfile) using the HBase source tree as
+# build context, then either:
+#
+# - Launches run_read_replica_integration_tests.sh inside that
+# container to execute the full test suite (default), or
+# - Starts a long-lived dev container for interactive exploration
+# (-d|--dev mode).
+#
+# The container uses Docker-outside-of-Docker (DooD) by bind-mounting
+# the host's Docker socket, and mounts .m2 for Maven cache reuse
+# (override the default $HOME location with -m|--m2).
+#
+# For usage information, run: ./hbase_nightly_read_replica_test.sh --help
set -e
REPLICA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-OUTPUT_DIR="${OUTPUT_DIR:-${REPLICA_DIR}/output}"
export HBASE_ROOT="$(cd "${REPLICA_DIR}/../.." && pwd)"
-export HBASE_IMAGE="hbase-read-replica:${BUILD_NUMBER:-local}"
-
-KEEP_IMAGE=false
-KEEP_CONTAINERS=false
+DEV_IMAGE_NAME="hbase-dev-support:${BUILD_NUMBER:-local}"
+
+PYTEST_K_VALUE=""
+JAVA_VERSION=""
+M2_DIR="${HOME}"
+EFFECTIVE_OUTPUT_DIR="${OUTPUT_DIR:-${REPLICA_DIR}/output}"
+DEV_MODE=false
+
+print_usage() {
+ SCRIPT=$(basename "${BASH_SOURCE}")
+
+ cat << __EOF
+
+hbase_nightly_read_replica_test.sh
+
+Outer driver script for the HBase read-replica integration test suite. Builds a
+dev-support Docker container image and either runs the inner test script
+(run_read_replica_integration_tests.sh) inside it, or starts a long-lived dev
+container for interactive exploration.
+
+Usage: ${SCRIPT} [options]
+
+ -h | --help Show this help message and exit.
+ -d | --dev Start a detached dev container for interactive
+ exploration instead of running the test suite.
+ Prints the container ID and instructions for
+ entering and stopping it.
+ -j | --java-version <ver> JVM version forwarded to the inner test script
+ (run_read_replica_integration_tests.sh), which
+ uses it to set JAVA_HOME inside the container.
+ -k <expression> Pytest -k filter expression forwarded to the
+ inner test script for test selection. (See
+ Pytest documentation on -k for more info.)
+ -m | --m2 <path> Parent directory of the .m2 Maven cache to
+ bind-mount into the container. Defaults to
+ \$HOME. The directory <path>/.m2 will be
+ created if it does not exist.
+
+__EOF
+}
while [[ $# -gt 0 ]]; do
case "$1" in
- -i|--keep-image)
- KEEP_IMAGE=true
- shift
+ -h|--help)
+ print_usage
+ exit 0
;;
- -c|--keep-containers)
- KEEP_CONTAINERS=true
+ -d|--dev)
+ DEV_MODE=true
shift
;;
+ -j|--java-version)
+ if [[ -n "$2" && "$2" != -* ]]; then
+ JAVA_VERSION="$2"
+ shift 2
+ else
+ echo "Error: Argument for $1 is missing" >&2
+ print_usage >&2
+ exit 1
+ fi
+ ;;
+ -k)
+ if [[ -n "$2" && "$2" != -* ]]; then
+ PYTEST_K_VALUE="$2"
+ shift 2
+ else
+ echo "Error: Argument for $1 is missing" >&2
+ print_usage >&2
+ exit 1
+ fi
+ ;;
+ -m|--m2)
+ if [[ -n "$2" && "$2" != -* ]]; then
+ M2_DIR="$2"
+ shift 2
+ else
+ echo "Error: Argument for $1 is missing" >&2
+ print_usage >&2
+ exit 1
+ fi
+ ;;
*)
- echo "Unknown option: $1"
- echo "Usage: $0 [-i|--keep-image] [-c|--keep-containers]"
+ echo "Unknown option: $1" >&2
+ print_usage >&2
exit 1
;;
esac
done
-echo "Replica dir: ${REPLICA_DIR}"
-echo "Output dir: ${OUTPUT_DIR}"
-echo "HBase root: ${HBASE_ROOT}"
-
-echo "Changing to replica dir: ${REPLICA_DIR}"
-cd "${REPLICA_DIR}"
-
-echo "Sourcing environment file: $(pwd)/.env"
-set -a
-source .env
-set +a
-
-echo "HBASE_IMAGE=${HBASE_IMAGE}"
-echo "ACTIVE_CLUSTER_CONF_DIR=${ACTIVE_CLUSTER_CONF_DIR}"
-echo "REPLICA_CLUSTER_CONF_DIR=${REPLICA_CLUSTER_CONF_DIR}"
-echo "DOCKER_COMPOSE_FILE=${DOCKER_COMPOSE_FILE}"
-echo "HBASE_DATA_STORE_ROOT=${HBASE_DATA_STORE_ROOT}"
-echo "realpath of HBASE_DATA_STORE_ROOT=$(realpath ${HBASE_DATA_STORE_ROOT})"
-
-# Clone HBase source for Docker build context (Docker COPY doesn't follow symlinks)
-echo "Cloning HBase source into ${REPLICA_DIR}/hbase for Docker build context..."
-rm -rf "${REPLICA_DIR}/hbase"
-git clone --local "${HBASE_ROOT}" "${REPLICA_DIR}/hbase"
-rm -rf "${REPLICA_DIR}/hbase/.git"
-
-cleanup() {
+echo "=== HBase Read-Replica Integration Test Driver ==="
+echo "HBase Root: ${HBASE_ROOT}"
+echo "Replica Dir: ${REPLICA_DIR}"
+echo "Dev Container Image: ${DEV_IMAGE_NAME}"
+echo "M2 Dir: ${M2_DIR}/.m2"
+echo "Output Dir: ${EFFECTIVE_OUTPUT_DIR}"
+
+# Build the dev-support container image using HBASE_ROOT as the build context
+echo "Building dev-support Docker image..."
+docker build --platform linux/amd64 \
+ -t "${DEV_IMAGE_NAME}" \
+ -f "${HBASE_ROOT}/dev-support/docker/Dockerfile" \
+ "${HBASE_ROOT}"
+
+cleanup_host() {
local exit_code=$?
- if [ ${exit_code} -ne 0 ]; then
- echo "=== FAILURE ==="
- echo "An error occurred during this stage in the Jenkins run."
- fi
- if [ "${KEEP_CONTAINERS}" = "false" ]; then
- echo "=== Cleanup: Stopping Docker containers ==="
- docker compose -f "${DOCKER_COMPOSE_FILE}" down 2>/dev/null || true
+ if [ -z "${BUILD_NUMBER}" ]; then
+ echo "Local execution complete. Preserving local container image ${DEV_IMAGE_NAME}."
else
- echo "=== Cleanup: Keeping Docker containers (--keep-containers) ==="
+ echo "Jenkins execution complete. Cleaning up image ${DEV_IMAGE_NAME}..."
+ docker rmi --force "${DEV_IMAGE_NAME}" 2>/dev/null || true
fi
- if [ "${KEEP_IMAGE}" = "false" ]; then
- echo "=== Cleanup: Removing Docker image: ${HBASE_IMAGE} ==="
- docker rmi --force "${HBASE_IMAGE}" 2>/dev/null || true
- else
- echo "=== Cleanup: Keeping Docker image: ${HBASE_IMAGE} (--keep-image) ==="
- fi
- echo "=== Cleanup: Deleting cloned HBase directory: ${REPLICA_DIR}/hbase ==="
- rm -rf "${REPLICA_DIR}/hbase"
exit "${exit_code}"
}
-trap cleanup EXIT
-
-# Copy latest proto file from source
-echo "Copying latest version of ActiveClusterSuffix.proto to $(pwd)/python/proto/"
-cp "${HBASE_ROOT}/hbase-protocol-shaded/src/main/protobuf/server/ActiveClusterSuffix.proto" \
- python/proto/
-
-export PYTHONPATH="$(pwd)"
-echo "Set PYTHONPATH=${PYTHONPATH}"
-
-# Create Python environment
-echo "Creating Python environment: .venv"
-python3 -m venv .venv
-source .venv/bin/activate
-
-# Install Python dependencies
-echo "Installing Python libraries"
-pip install --upgrade pip
-pip install -r requirements.txt
-
-# Compile protobuf
-echo "Compiling Protobuf"
-python3 python/proto/proto_compiler.py
-
-# Build Docker images
-echo "Building hbase-docker image"
-./build-images.sh
-
-# Run read-replica integration test suite
-echo "Starting read-replica integration test scripts"
-echo "Starting read-replica integration test suite via Pytest..."
-pytest --html="${OUTPUT_DIR}/read-replica-nightly-test-report.html" \
- --self-contained-html \
- --junitxml="${OUTPUT_DIR}/read-replica-nightly-test-results.xml" \
- python/test/test_read_replica_feature.py
-
-echo "=== Success: All read-replica integration tests passed. ==="
+trap cleanup_host EXIT
+
+# Ensure host .m2 directory exists for caching
+mkdir -p "${M2_DIR}/.m2"
+mkdir -p "${EFFECTIVE_OUTPUT_DIR}"
+
+if [ "${DEV_MODE}" = "true" ]; then
+ # Start a detached dev container for interactive exploration
+ echo "Starting dev container in background..."
+ CONTAINER_ID=$(docker run -d \
+ --platform linux/amd64 \
+ -v /var/run/docker.sock:/var/run/docker.sock \
+ -v "${HBASE_ROOT}:${HBASE_ROOT}" \
+ -v "${EFFECTIVE_OUTPUT_DIR}:${EFFECTIVE_OUTPUT_DIR}" \
+ -v "${M2_DIR}/.m2:/root/.m2" \
+ -e OUTPUT_DIR="${EFFECTIVE_OUTPUT_DIR}" \
+ -e BUILD_NUMBER="${BUILD_NUMBER:-local}" \
+ -w "${REPLICA_DIR}" \
+ "${DEV_IMAGE_NAME}" \
+ sleep infinity)
+
+ echo ""
+ echo "=== Dev container is ready ==="
+ echo "Container ID: ${CONTAINER_ID}"
+ echo "Image: ${DEV_IMAGE_NAME}"
+ echo ""
+ echo "Enter the container:"
+ echo ""
+ echo "docker exec -it ${CONTAINER_ID} bash"
+ echo ""
+ echo "Stop and remove the container when done:"
+ echo ""
+ echo "docker stop ${CONTAINER_ID} && docker rm ${CONTAINER_ID}"
+ echo ""
+else
+ JAVA_VERSION_ARGS=()
+ if [[ -n "${JAVA_VERSION}" ]]; then
+ JAVA_VERSION_ARGS=(-j "${JAVA_VERSION}")
+ fi
+
+ PYTEST_K_ARGS=()
+ if [[ -n "${PYTEST_K_VALUE}" ]]; then
+ PYTEST_K_ARGS=(-k "${PYTEST_K_VALUE}")
+ fi
+
+ # Run the inner test script inside the dev-support container via DooD
+ echo "Launching dev container and starting test suite..."
+ docker run --rm \
+ --platform linux/amd64 \
+ -v /var/run/docker.sock:/var/run/docker.sock \
+ -v "${HBASE_ROOT}:${HBASE_ROOT}" \
+ -v "${EFFECTIVE_OUTPUT_DIR}:${EFFECTIVE_OUTPUT_DIR}" \
+ -v "${M2_DIR}/.m2:/root/.m2" \
+ -e OUTPUT_DIR="${EFFECTIVE_OUTPUT_DIR}" \
+ -e BUILD_NUMBER="${BUILD_NUMBER:-local}" \
+ -w "${REPLICA_DIR}" \
+ "${DEV_IMAGE_NAME}" \
+ ./run_read_replica_integration_tests.sh "${JAVA_VERSION_ARGS[@]}" "${PYTEST_K_ARGS[@]}"
+fi
diff --git a/dev-support/read-replica/hbase_nightly_read_replica_test.sh b/dev-support/read-replica/run_read_replica_integration_tests.sh
similarity index 54%
copy from dev-support/read-replica/hbase_nightly_read_replica_test.sh
copy to dev-support/read-replica/run_read_replica_integration_tests.sh
index b6bec403ced..d4b67e11230 100755
--- a/dev-support/read-replica/hbase_nightly_read_replica_test.sh
+++ b/dev-support/read-replica/run_read_replica_integration_tests.sh
@@ -15,9 +15,24 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-# Run the read-replica Docker integration test suite.
+# run_read_replica_integration_tests.sh
+#
+# Inner test runner executed INSIDE the Docker container built from
+# hbase/dev-support/docker/Dockerfile. In a typical test run, this
+# script is invoked by hbase_nightly_read_replica_test.sh. It can be
+# run on its own as well as long as it is done within the container
+# mentioned above.
+#
+# What it does:
+# 1. Clones the HBase source tree into a local directory for Docker build context
+# 2. Copies and compiles the protobuf definitions needed by the Python tests
+# 3. Creates a Python virtual environment and installs dependencies
+# 4. Builds a Docker image for the active and replica clusters in a read-replica
+# setup
+# 5. Runs the pytest read-replica integration suite using these clusters
+#
+# For usage information, run: ./run_read_replica_integration_tests.sh --help
set -e
REPLICA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -26,11 +41,55 @@ export HBASE_ROOT="$(cd "${REPLICA_DIR}/../.." && pwd)"
export HBASE_IMAGE="hbase-read-replica:${BUILD_NUMBER:-local}"
+JAVA_VERSION=17
KEEP_IMAGE=false
KEEP_CONTAINERS=false
+PYTEST_K_VALUE=""
+
+print_usage() {
+ SCRIPT=$(basename "${BASH_SOURCE}")
+
+ cat << __EOF
+
+run_read_replica_integration_tests.sh
+
+Inner test runner executed inside the Docker container built from
+hbase/dev-support/docker/Dockerfile. Clones the HBase source, builds Docker
+images for the active and read-replica clusters, and runs the pytest
+integration suite.
+
+This script is normally invoked by hbase_nightly_read_replica_test.sh, but
+it can also be run on its own within the container mentioned above.
+
+Usage: ${SCRIPT} [options]
+
+ -h | --help Show this help message and exit.
+ -j | --java-version <ver> JVM version to use (default: 17). Sets JAVA_HOME
+ to /usr/lib/jvm/java-<ver>.
+ -i | --keep-image Do not remove the Docker image on exit.
+ -c | --keep-containers Do not run 'docker compose down' on exit.
+ -k <expression> Pytest -k filter expression for test selection.
+ (See Pytest documentation on -k for more info.)
+
+__EOF
+}
while [[ $# -gt 0 ]]; do
case "$1" in
+ -h|--help)
+ print_usage
+ exit 0
+ ;;
+ -j|--java-version)
+ if [[ -n "$2" && "$2" != -* ]]; then
+ JAVA_VERSION="$2"
+ shift 2
+ else
+ echo "Error: Argument for $1 is missing" >&2
+ print_usage >&2
+ exit 1
+ fi
+ ;;
-i|--keep-image)
KEEP_IMAGE=true
shift
@@ -39,14 +98,32 @@ while [[ $# -gt 0 ]]; do
KEEP_CONTAINERS=true
shift
;;
+ -k)
+ if [[ -n "$2" && "$2" != -* ]]; then
+ PYTEST_K_VALUE="$2"
+ shift 2
+ else
+ echo "Error: Argument for $1 is missing" >&2
+ print_usage >&2
+ exit 1
+ fi
+ ;;
*)
- echo "Unknown option: $1"
- echo "Usage: $0 [-i|--keep-image] [-c|--keep-containers]"
+ echo "Unknown option: $1" >&2
+ print_usage >&2
exit 1
;;
esac
done
+# Set JAVA_HOME and update PATH for the selected JVM (e.g. 8, 11, 17, 21)
+# MAVEN_HOME is set in hbase/dev-support/docker/Dockerfile
+export JAVA_HOME="/usr/lib/jvm/java-${JAVA_VERSION}"
+export PATH="${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${PATH}"
+
+echo "Using JAVA_HOME=${JAVA_HOME}"
+
+echo "=== Inside Dev Container Environment ==="
echo "Replica dir: ${REPLICA_DIR}"
echo "Output dir: ${OUTPUT_DIR}"
echo "HBase root: ${HBASE_ROOT}"
@@ -76,7 +153,7 @@ cleanup() {
local exit_code=$?
if [ ${exit_code} -ne 0 ]; then
echo "=== FAILURE ==="
- echo "An error occurred during this stage in the Jenkins run."
+ echo "An error occurred during this stage of the run."
fi
if [ "${KEEP_CONTAINERS}" = "false" ]; then
echo "=== Cleanup: Stopping Docker containers ==="
@@ -123,11 +200,19 @@ echo "Building hbase-docker image"
./build-images.sh
# Run read-replica integration test suite
-echo "Starting read-replica integration test scripts"
+PYTEST_K_ARGS=()
+if [[ -n "${PYTEST_K_VALUE}" ]]; then
+ PYTEST_K_ARGS=(-k "${PYTEST_K_VALUE}")
+fi
+
echo "Starting read-replica integration test suite via Pytest..."
-pytest --html="${OUTPUT_DIR}/read-replica-nightly-test-report.html" \
+pytest -o log_cli=true --log-cli-level=INFO \
+ --log-cli-format='%(asctime)s %(levelname)-5s %(module)s.%(funcName)s(%(lineno)d): %(message)s' \
+ --log-cli-date-format='%Y-%m-%d %H:%M:%S' \
+ --html="${OUTPUT_DIR}/read-replica-nightly-test-report.html" \
--self-contained-html \
--junitxml="${OUTPUT_DIR}/read-replica-nightly-test-results.xml" \
- python/test/test_read_replica_feature.py
+ python/test/test_read_replica_feature.py \
+ "${PYTEST_K_ARGS[@]}"
echo "=== Success: All read-replica integration tests passed. ==="