You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/10/20 11:54:04 UTC
(iggy) branch ci/publish-kafka-gateway updated: fix(ci): isolate image releases and enforce SDK tag checks
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch ci/publish-kafka-gateway
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/ci/publish-kafka-gateway by this push:
new 0d02348f5 fix(ci): isolate image releases and enforce SDK tag checks
0d02348f5 is described below
commit 0d02348f51a8c3200801d5cf6de83f1525f76f54
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Tue Oct 20 13:40:42 2026 +0200
fix(ci): isolate image releases and enforce SDK tag checks
One failed image build blocked every manifest, while SDK publishing
could bypass failed tag validation and non-server cache exports
overwrote the server cache.
Require validated SDK tags and both architecture artifacts per image.
Keep registry cache writes scoped to each image. Install gateway
license tooling early with retries, register version bumps, and
correct the container documentation.
---
.github/actions/utils/docker-buildx/action.yml | 1 -
.github/workflows/publish.yml | 47 ++++++--------------------
gateways/kafka/Dockerfile | 16 ++++++---
gateways/kafka/README.md | 42 +++++++++++------------
gateways/kafka/docker-compose.yml | 7 ++--
gateways/kafka/docs/CONSUMER_GROUPS.md | 5 +--
gateways/kafka/docs/MANUAL_TESTING.md | 2 +-
scripts/bump-version.sh | 6 +++-
8 files changed, 54 insertions(+), 72 deletions(-)
diff --git a/.github/actions/utils/docker-buildx/action.yml b/.github/actions/utils/docker-buildx/action.yml
index 6c91c9966..99d8fb3e8 100644
--- a/.github/actions/utils/docker-buildx/action.yml
+++ b/.github/actions/utils/docker-buildx/action.yml
@@ -255,7 +255,6 @@ runs:
else
CACHE_TO=$(
printf '%s\n' \
- "type=registry,ref=${shared_deps}:buildcache-${arch},mode=max" \
"type=registry,ref=${img}:buildcache-${arch},mode=max" \
"type=inline" \
"type=gha,scope=buildkit-deps-${os}-${arch},mode=max" \
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 9ee38aba5..aa55cbe96 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -375,12 +375,7 @@ jobs:
run: |
/tmp/copy-latest-from-master.sh save \
.github \
- scripts \
- web/Dockerfile \
- core/server/Dockerfile \
- core/ai/mcp/Dockerfile \
- core/connectors/runtime/Dockerfile \
- core/bench/dashboard/server/Dockerfile
+ scripts
/tmp/copy-latest-from-master.sh apply
@@ -705,17 +700,14 @@ jobs:
- name: Save and apply latest CI from master
if: inputs.use_latest_ci
+ env:
+ DOCKERFILE: ${{ matrix.dockerfile }}
run: |
/tmp/copy-latest-from-master.sh save \
.github \
scripts \
- web/Dockerfile \
- core/server/Dockerfile \
- core/ai/mcp/Dockerfile \
- core/connectors/runtime/Dockerfile \
- core/bench/dashboard/server/Dockerfile \
- gateways/kafka/Dockerfile \
- gateways/kafka/Dockerfile.dockerignore
+ "$DOCKERFILE" \
+ "$DOCKERFILE.dockerignore"
/tmp/copy-latest-from-master.sh apply
@@ -784,7 +776,7 @@ jobs:
needs.validate.outputs.has_targets == 'true' &&
needs.plan.outputs.has_docker == 'true' &&
fromJson(needs.plan.outputs.docker_components).include[0].key != 'noop' &&
- needs.publish-docker.result == 'success' &&
+ (needs.publish-docker.result == 'success' || needs.publish-docker.result == 'failure') &&
inputs.dry_run == false
runs-on: ubuntu-26.04
strategy:
@@ -873,21 +865,6 @@ jobs:
} >> "$GITHUB_OUTPUT"
echo "✅ Resolved $MATRIX_KEY -> version=$VERSION tag=${TAG:-<none>} should_tag=$SHOULD_TAG"
- - name: Resolve image from config
- id: config
- shell: bash
- run: |
- if ! command -v yq >/dev/null 2>&1; then
- YQ_VERSION="v4.47.1"
- YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb"
- curl -fsSL --retry 5 --retry-all-errors -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64
- echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1
- chmod +x /usr/local/bin/yq
- fi
- image=$(yq ".components.${{ matrix.key }}.image" .github/config/publish.yml)
- echo "image=$image" >> "$GITHUB_OUTPUT"
- echo "📦 Image: $image"
-
- name: Download amd64 digest
uses: actions/download-artifact@v8
with:
@@ -912,9 +889,9 @@ jobs:
- name: Create and push manifest
working-directory: ${{ runner.temp }}/digests
env:
+ IMAGE: ${{ matrix.image }}
SHOULD_TAG: ${{ steps.ver.outputs.should_tag }}
run: |
- IMAGE="${{ steps.config.outputs.image }}"
VERSION="${{ steps.ver.outputs.version }}"
echo "Creating manifests for $IMAGE from digests:"
@@ -959,9 +936,9 @@ jobs:
- name: Inspect manifest
env:
+ IMAGE: ${{ matrix.image }}
SHOULD_TAG: ${{ steps.ver.outputs.should_tag }}
run: |
- IMAGE="${{ steps.config.outputs.image }}"
VERSION="${{ steps.ver.outputs.version }}"
# Auto-publish pushed :version only when should_tag was true; in
@@ -1001,6 +978,7 @@ jobs:
needs: [validate, plan, check-tags, build-python-wheels, publish-rust-crates]
if: |
!cancelled() &&
+ needs.check-tags.result == 'success' &&
needs.validate.outputs.has_targets == 'true' &&
fromJson(needs.plan.outputs.non_docker_targets).include[0].key != 'noop' &&
(needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped') &&
@@ -1040,12 +1018,7 @@ jobs:
run: |
/tmp/copy-latest-from-master.sh save \
.github \
- scripts \
- web/Dockerfile \
- core/server/Dockerfile \
- core/ai/mcp/Dockerfile \
- core/connectors/runtime/Dockerfile \
- core/bench/dashboard/server/Dockerfile
+ scripts
/tmp/copy-latest-from-master.sh apply
diff --git a/gateways/kafka/Dockerfile b/gateways/kafka/Dockerfile
index c73837f88..a4451e8f9 100644
--- a/gateways/kafka/Dockerfile
+++ b/gateways/kafka/Dockerfile
@@ -30,6 +30,14 @@ RUN apt-get update && apt-get install -y \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
+RUN TARGET="$(uname -m)-unknown-linux-musl" && \
+ tarball="$(mktemp)" && \
+ curl -sSfL --retry 5 --retry-all-errors -o "$tarball" \
+ "https://github.com/EmbarkStudios/cargo-about/releases/download/0.9.0/cargo-about-0.9.0-${TARGET}.tar.gz" && \
+ tar -xzf "$tarball" -C /usr/local/bin --strip-components=1 \
+ "cargo-about-0.9.0-${TARGET}/cargo-about" && \
+ rm -f "$tarball"
+
COPY rust-toolchain.toml rust-toolchain.toml
# Resolves/installs the toolchain now, in a layer keyed only on rust-toolchain.toml - matching
# RUST_VERSION above means this is a no-op resolution, not a download, but isolates the cost
@@ -46,10 +54,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
cargo build --release --locked --package iggy-gateway-kafka --bin iggy-gateway-kafka && \
cp target/release/iggy-gateway-kafka /iggy-gateway-kafka
+# ASF convenience binaries must include the license text of bundled third-party code.
RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
- TARGET="$(uname -m)-unknown-linux-musl" && \
- curl -sSfL "https://github.com/EmbarkStudios/cargo-about/releases/download/0.9.0/cargo-about-0.9.0-${TARGET}.tar.gz" \
- | tar -xz -C /usr/local/bin --strip-components=1 "cargo-about-0.9.0-${TARGET}/cargo-about" && \
./scripts/ci/third-party-licenses.sh --generate \
--manifest gateways/kafka/Cargo.toml \
--output /build/LICENSE-binary
@@ -62,8 +68,8 @@ RUN apt-get update && apt-get install -y \
netcat-openbsd \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /iggy-gateway-kafka .
-COPY --from=builder /build/LICENSE-binary /usr/share/doc/iggy/LICENSE-binary
-COPY LICENSE NOTICE /usr/share/doc/iggy/
+COPY --from=builder /build/LICENSE-binary /usr/share/doc/iggy-kafka-gateway/LICENSE-binary
+COPY LICENSE NOTICE /usr/share/doc/iggy-kafka-gateway/
ENV IGGY_KAFKA_BIND_ADDR=0.0.0.0:9093
# No baked IGGY_KAFKA_ADVERTISED_HOST default: wildcard bind means the gateway cannot derive
diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md
index 682ac83ca..b123adcd7 100644
--- a/gateways/kafka/README.md
+++ b/gateways/kafka/README.md
@@ -24,8 +24,9 @@ Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/34
> rebalances, graceful leave, session expiry and admin views of that membership
> ([docs/CONSUMER_GROUPS.md](docs/CONSUMER_GROUPS.md)). `kafka-consumer-groups.sh --list`,
> `--describe --members` and `--describe --state` use those views. The default `--describe`
-> calls OffsetFetch (API key 9), which is not implemented, so that mode does not work. With the bridge off,
-> Metadata reports every topic unknown, so a consumer joins a group and is assigned 0 partitions.
+> calls OffsetFetch (API key 9), which requires the bridge and otherwise answers
+> `COORDINATOR_LOAD_IN_PROGRESS` (14). With the bridge off, Metadata reports every topic unknown,
+> so a consumer joins a group and is assigned 0 partitions.
> With the bridge on, OffsetCommit and OffsetFetch keep each group's offsets in Iggy, so a group
> resumes from its last commit, also after a gateway restart
> ([docs/OFFSET_STORAGE.md](docs/OFFSET_STORAGE.md)). Every Iggy server must run a build with the
@@ -76,17 +77,22 @@ connectors runtime:
| Tag | Publishing rule |
| --- | --- |
-| `edge` | Rolling build refreshed after affected changes merge to `master`. |
+| `edge` | Rolling build refreshed by successful publishing after affected changes merge to `master`. It can lag after a failed publish run. |
| `<version>` | Version from `gateways/kafka/Cargo.toml`. Stable releases are published manually; new prereleases can be published automatically. |
| `latest` | Updated only by a manual stable release. |
-Automatic publishing of a stable version only refreshes `edge`; it does not update the
-versioned tag or `latest`.
+CI treats only versions containing `-edge` or `-rc` as prereleases, such as `0.2.0-edge.1`
+or `0.2.0-rc.1`.
```bash
docker pull apache/iggy-kafka-gateway:edge
```
+With the image's default listener, set `IGGY_KAFKA_ADVERTISED_HOST` to a host reachable by
+Kafka clients; see [Run](#run). Produce and Fetch need `IGGY_KAFKA_BRIDGE_ENABLED=true`
+and the Iggy connection settings. The [Compose quick start](#quick-start-docker-compose)
+sets both the advertised address and the bridge configuration.
+
## Quick start (Docker Compose)
The fastest way to see a Kafka client talk to Iggy: two containers, no local Rust toolchain to
@@ -103,7 +109,7 @@ cd gateways/kafka
docker compose up --build
```
-By default, `iggy-server` pulls the prebuilt `apache/iggy:edge` image (refreshed on a master push only when `server` or `iggy-cli` is affected, or `web` or `core/server/Dockerfile` changes),
+By default, `iggy-server` pulls the prebuilt `apache/iggy:edge` image,
so only `iggy-gateway-kafka` compiles from source. Its Dockerfile mounts a cargo registry/target
cache, so a first run costs one real compile and every run after reuses it, even across a source
change elsewhere in the workspace. Testing a local `iggy-server` change: comment out `image:` and
@@ -111,15 +117,8 @@ uncomment `build:` in `docker-compose.yml` first. That build mounts cargo regist
target caches, so the first run is still a full release build of several minutes and later runs
reuse those mounts.
-To use the prebuilt gateway too, replace the `build:` block under `iggy-gateway-kafka` in
-[`docker-compose.yml`](docker-compose.yml) with:
-
-```yaml
- image: apache/iggy-kafka-gateway:edge
- pull_policy: always
-```
-
-Keep the service's other settings, including its bridge configuration and advertised address.
+To use the prebuilt gateway too, comment out its `build:` block in
+[`docker-compose.yml`](docker-compose.yml) and uncomment its `image:` and `pull_policy:` lines.
The [`pull_policy: always`](https://docs.docker.com/reference/compose-file/services/#pull_policy)
setting checks for a newer rolling image each time. From `gateways/kafka`, start both images
without compiling:
@@ -128,6 +127,9 @@ without compiling:
docker compose up --no-build
```
+Before testing local gateway changes or running [manual testing](docs/MANUAL_TESTING.md)
+Category I, restore the `build:` block and comment out `image:` and `pull_policy:`.
+
This starts `iggy-server` (the real Iggy broker) and `iggy-gateway-kafka` (bridge enabled,
pointed at that server) on a shared Docker network, with the gateway waiting for the server's
health check before it starts. From another terminal:
@@ -178,9 +180,7 @@ stack. See `gateways/kafka/docker-compose.yml` and the environment variable tabl
every other knob.
**Limitations** (see the stub warning above for the full list): single gateway, single Iggy node,
-SASL off by default. This is a development quick start, not a production deployment shape. If your
-local `apache/iggy:edge` predates [#4392](https://github.com/apache/iggy/pull/4392), run
-`docker compose pull`, because that server refuses every offset commit.
+SASL off by default. This is a development quick start, not a production deployment shape.
This quick start's topics use Iggy's default `Durability::Replicated` - a Produce ack is a
quorum commit, not an fsync. A `docker kill` or host power loss can lose acked records despite
@@ -278,8 +278,6 @@ Limits:
- Classic protocol only. A consumer with `group.protocol=consumer` (KIP-848) fails.
- `group.instance.id` is accepted but not honoured. A static member that restarts causes two
rebalances.
-- No DescribeGroups or ListGroups yet ([#3548](https://github.com/apache/iggy/issues/3548)), so
- `kafka-consumer-groups.sh` cannot list or describe a group.
- No DeleteGroups or OffsetDelete, and offsets never expire. An operator deletes a group's offsets
in Iggy ([docs/OFFSET_STORAGE.md](docs/OFFSET_STORAGE.md#limits)).
- Delivery is at-least-once. A consumer that stops before it commits reads those records again.
@@ -439,9 +437,9 @@ topic per call) and records by time for `ListOffsets`, probes and polls partitio
stores and reads group offsets for OffsetCommit and OffsetFetch.
Produce ([#3535](https://github.com/apache/iggy/issues/3535)), Fetch ([#3536](https://github.com/apache/iggy/issues/3536)), ListOffsets
([#3537](https://github.com/apache/iggy/issues/3537)), Metadata ([#3534](https://github.com/apache/iggy/issues/3534)), CreateTopics
-([#3538](https://github.com/apache/iggy/issues/3538)), DescribeConfigs and AlterConfigs , OffsetCommit and OffsetFetchcall it.
+([#3538](https://github.com/apache/iggy/issues/3538)), DescribeConfigs, AlterConfigs, OffsetCommit
+and OffsetFetch ([#3542](https://github.com/apache/iggy/issues/3542)) call it.
Topic configuration behavior is in [docs/CONFIGS.md](docs/CONFIGS.md).
-([#3542](https://github.com/apache/iggy/issues/3542)) call it.
Tested by `bridge`'s own unit tests, `tests/bridge_iggy_integration_tests.rs` and the
`tests/*_real_bridge_tests.rs` suites. All but the unit tests start a real `iggy-server`.
diff --git a/gateways/kafka/docker-compose.yml b/gateways/kafka/docker-compose.yml
index fff7bcb65..9d561a2c8 100644
--- a/gateways/kafka/docker-compose.yml
+++ b/gateways/kafka/docker-compose.yml
@@ -20,9 +20,8 @@
services:
iggy-server:
- # Prebuilt. :edge refreshes on a master push only when `server` or `iggy-cli` is
- # affected, or `web` or `core/server/Dockerfile` changes - avoids a full from-scratch workspace
- # release build on first `up --build`. Testing a local server change: comment this out
+ # Prebuilt. :edge refreshes after affected changes merge to master.
+ # Testing a local server change: comment this out
# and uncomment `build:` below instead - `core/server/Dockerfile`'s `runtime` target is
# the same Dockerfile and target `apache/iggy:edge` itself is published from
# (.github/config/publish.yml), so the healthcheck/volume/listener-address settings below
@@ -93,6 +92,8 @@ services:
retries: 30
iggy-gateway-kafka:
+ # image: apache/iggy-kafka-gateway:edge
+ # pull_policy: always
build:
context: ../..
dockerfile: gateways/kafka/Dockerfile
diff --git a/gateways/kafka/docs/CONSUMER_GROUPS.md b/gateways/kafka/docs/CONSUMER_GROUPS.md
index 166f6acac..92798d7ae 100644
--- a/gateways/kafka/docs/CONSUMER_GROUPS.md
+++ b/gateways/kafka/docs/CONSUMER_GROUPS.md
@@ -48,8 +48,9 @@ second snapshot of the same members would otherwise multiply the encoded respons
count.
`kafka-consumer-groups.sh --list`, `--describe --members` and `--describe --state` use these
-two APIs. The default `--describe` calls `OffsetFetch` (API key 9), which is not implemented,
-so that mode does not work. `--reset-offsets` needs the same API.
+two APIs. The default `--describe` and `--reset-offsets` use `OffsetFetch` (API key 9),
+which requires the bridge. Without it, OffsetFetch answers `COORDINATOR_LOAD_IN_PROGRESS`
+(14). See [OFFSET_STORAGE.md](OFFSET_STORAGE.md) for offset behavior and limits.
Member `client_id` and `client_host` are empty. The coordinator does not keep the request
header's client id or the connection's peer address. `include_authorized_operations` is answered
diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md
index 0bf1dab07..a670c49a6 100644
--- a/gateways/kafka/docs/MANUAL_TESTING.md
+++ b/gateways/kafka/docs/MANUAL_TESTING.md
@@ -458,7 +458,7 @@ success. This category predates the bridge landing: G2/G3's "fails at metadata"
| G11 | Two members share a topic | Bridge on and a topic with 2 partitions. Run `kafka-console-consumer.sh --bootstrap-server 127.0.0.1:9093 --group g11 --topic <topic> --property print.partition=true --property print.offset=true` in two terminals, produce to both partitions, stop one, produce again | Each prints one partition. After the stop, the other prints the new records of both partitions, and none twice |
| G12 | List groups | `kafka-consumer-groups.sh --bootstrap-server 127.0.0.1:9093 --list` after G5 has joined | Prints the group. `--list --state Empty` matches a group that was issued a member id and has not joined yet. A group whose members have all expired is absent |
| G13 | Describe members | `kafka-consumer-groups.sh --bootstrap-server 127.0.0.1:9093 --describe --members --group g2` | Prints member ids. Assignment and protocol are filled only while the group is Stable |
-| G14 | Describe state | `kafka-consumer-groups.sh --bootstrap-server 127.0.0.1:9093 --describe --state --group g2` | Prints the state string (`Stable`, `PreparingRebalance`, `CompletingRebalance`, or `Empty`). The default `--describe` (no `--members` or `--state`) calls OffsetFetch and fails, because that API is not implemented |
+| G14 | Describe state | `kafka-consumer-groups.sh --bootstrap-server 127.0.0.1:9093 --describe --state --group g2` | Prints the state string (`Stable`, `PreparingRebalance`, `CompletingRebalance`, or `Empty`). The default `--describe` (no `--members` or `--state`) calls OffsetFetch, which requires the bridge and otherwise answers `COORDINATOR_LOAD_IN_PROGRESS` (14) |
Record kcat version and exact error strings in your test log. G1 passing is the minimum bar for client compatibility smoke.
diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh
index d64ffe8c9..0ef47a5d9 100755
--- a/scripts/bump-version.sh
+++ b/scripts/bump-version.sh
@@ -40,6 +40,7 @@ Components:
rust-cli core/cli + workspace dep
rust-connector-sdk core/connectors/sdk + workspace dep
rust-mcp core/ai/mcp
+ rust-kafka-gateway gateways/kafka
rust-bench core/bench
rust-bench-dashboard-frontend core/bench/dashboard/frontend
rust-bench-dashboard-server core/bench/dashboard/server
@@ -86,7 +87,7 @@ Examples:
EOF
}
-RUST_COMPONENTS="rust-sdk rust-common rust-binary-protocol rust-server rust-cli rust-connector-sdk rust-mcp rust-bench rust-bench-dashboard-frontend rust-bench-dashboard-server rust-bench-report"
+RUST_COMPONENTS="rust-sdk rust-common rust-binary-protocol rust-server rust-cli rust-connector-sdk rust-mcp rust-kafka-gateway rust-bench rust-bench-dashboard-frontend rust-bench-dashboard-server rust-bench-report"
CONNECTOR_SINK_COMPONENTS="rust-connector-clickhouse-sink rust-connector-delta-sink rust-connector-doris-sink rust-connector-elasticsearch-sink rust-connector-http-sink rust-connector-iceberg-sink rust-connector-influxdb-sink rust-connector-meilisearch-sink rust-connector-mongodb-sink rust-connector-postgres-sink rust-connector-quickwit-sink rust-connector-rabbitmq-sink rust-connector-redshift-sink rust-connector-s3-sink rust-connector-stdout-sink rust-connector-surrealdb-sink"
CONNECTOR_SOURCE_COMPONENTS="rust-connector-elasticsearch-source rust-connector-http-source rust-connector-influxdb-source rust-connector-postgres-source rust-connector-random-source"
CONNECTOR_COMPONENTS="rust-connector-runtime ${CONNECTOR_SINK_COMPONENTS} ${CONNECTOR_SOURCE_COMPONENTS}"
@@ -136,6 +137,9 @@ get_version_files() {
rust-mcp)
echo "core/ai/mcp/Cargo.toml:cargo"
;;
+ rust-kafka-gateway)
+ echo "gateways/kafka/Cargo.toml:cargo"
+ ;;
rust-bench)
echo "core/bench/Cargo.toml:cargo"
;;