You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2025/12/15 11:59:33 UTC
(iggy) branch refactor-post-merge-publish created (now 9de452c10)
This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a change to branch refactor-post-merge-publish
in repository https://gitbox.apache.org/repos/asf/iggy.git
at 9de452c10 feat(ci): replace Docker buildx with native runner builds
This branch includes the following new commits:
new 9de452c10 feat(ci): replace Docker buildx with native runner builds
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iggy) 01/01: feat(ci): replace Docker buildx with native runner builds
Posted by hg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a commit to branch refactor-post-merge-publish
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 9de452c10b75d27fbd9b074fb9c6749f3ffdfaaf
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Mon Dec 15 12:58:48 2025 +0100
feat(ci): replace Docker buildx with native runner builds
No more QEMU emulation for Rust images. Changes:
- Add _build_docker_images.yml workflow using native ARM/x86 runners
- Add prebuilt stages to mcp, connectors, bench-dashboard Dockerfiles
- Extend _build_rust_artifacts.yml with iggy-mcp and bench-dashboard
- Update post-merge.yml and publish.yml to use new workflow
- Deprecate docker-buildx action (kept only for web-ui)
---
.github/actions/utils/docker-buildx/action.yml | 16 +-
.github/workflows/_build_docker_images.yml | 409 +++++++++++++++++++++++++
.github/workflows/_build_rust_artifacts.yml | 98 +++++-
.github/workflows/post-merge.yml | 80 ++---
.github/workflows/publish.yml | 127 +++++++-
core/ai/mcp/Dockerfile | 21 +-
core/bench/dashboard/server/Dockerfile | 62 +++-
core/connectors/runtime/Dockerfile | 21 +-
8 files changed, 749 insertions(+), 85 deletions(-)
diff --git a/.github/actions/utils/docker-buildx/action.yml b/.github/actions/utils/docker-buildx/action.yml
index 276cb6e0b..ba34b5f39 100644
--- a/.github/actions/utils/docker-buildx/action.yml
+++ b/.github/actions/utils/docker-buildx/action.yml
@@ -15,8 +15,22 @@
# specific language governing permissions and limitations
# under the License.
+# ==================================================================================
+# DEPRECATION NOTICE
+# ==================================================================================
+# This action uses Docker buildx with QEMU emulation for multi-arch builds.
+# For Rust components (server, mcp, connectors, bench-dashboard), use the new
+# _build_docker_images.yml workflow instead, which uses native runners for each
+# architecture, resulting in ~4x faster builds.
+#
+# This action is ONLY used for:
+# - web-ui (Node.js) - QEMU overhead is minimal for Node.js builds
+#
+# Do NOT use this action for new Rust Docker images.
+# ==================================================================================
+
name: docker-buildx
-description: Multi-arch Docker build and push
+description: Multi-arch Docker build and push (DEPRECATED for Rust images - use _build_docker_images.yml instead)
inputs:
task:
description: "Task to run (build/publish/docker)"
diff --git a/.github/workflows/_build_docker_images.yml b/.github/workflows/_build_docker_images.yml
new file mode 100644
index 000000000..3846a5748
--- /dev/null
+++ b/.github/workflows/_build_docker_images.yml
@@ -0,0 +1,409 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: _build_docker_images
+on:
+ workflow_call:
+ inputs:
+ version:
+ type: string
+ required: false
+ default: "edge"
+ description: "Version tag for Docker images"
+ dry_run:
+ type: boolean
+ required: false
+ default: false
+ description: "If true, build images but don't push"
+ components:
+ type: string
+ required: false
+ default: "rust-server,rust-mcp,rust-connectors,rust-bench-dashboard"
+ description: "Comma-separated list of components to build"
+
+env:
+ IGGY_CI_BUILD: true
+
+jobs:
+ build-images:
+ name: ${{ matrix.component }} (${{ matrix.arch }})
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ arch: [amd64, arm64]
+ # Note: web-ui is NOT included here - it's a Node.js project that doesn't need
+ # pre-built Rust artifacts. It continues to use docker-buildx action directly.
+ component: [rust-server, rust-mcp, rust-connectors, rust-bench-dashboard]
+ include:
+ - arch: amd64
+ runner: ubuntu-latest
+ artifact_target: x86_64-unknown-linux-gnu
+ - arch: arm64
+ runner: ubuntu-24.04-arm
+ artifact_target: aarch64-unknown-linux-gnu
+ steps:
+ - name: Check if component should be built
+ id: check
+ run: |
+ components="${{ inputs.components }}"
+ component="${{ matrix.component }}"
+ # Use word boundary matching to avoid partial matches
+ if echo "$components" | grep -wq "$component"; then
+ echo "should_build=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "should_build=false" >> "$GITHUB_OUTPUT"
+ echo "Skipping $component (not in component list)"
+ fi
+
+ - name: Checkout code
+ if: steps.check.outputs.should_build == 'true'
+ uses: actions/checkout@v4
+
+ - name: Download binaries artifact
+ if: steps.check.outputs.should_build == 'true' && matrix.component != 'rust-bench-dashboard'
+ uses: actions/download-artifact@v4
+ with:
+ name: binaries-${{ matrix.artifact_target }}
+ path: artifacts
+
+ - name: Download bench-dashboard artifact
+ if: steps.check.outputs.should_build == 'true' && matrix.component == 'rust-bench-dashboard'
+ uses: actions/download-artifact@v4
+ with:
+ name: bench-dashboard-${{ matrix.artifact_target }}
+ path: artifacts
+
+ - name: Extract artifacts
+ if: steps.check.outputs.should_build == 'true'
+ run: |
+ cd artifacts
+ # Extract all tarballs (handles single or multiple files safely)
+ for tarball in *.tar.gz; do
+ [ -f "$tarball" ] || continue
+ echo "Extracting $tarball..."
+ tar xzf "$tarball"
+ rm -f "$tarball"
+ done
+ echo "Extracted artifacts:"
+ ls -la
+
+ - name: Set up Docker
+ if: steps.check.outputs.should_build == 'true'
+ run: |
+ # Ensure docker is available
+ docker version
+
+ - name: Login to Docker Hub
+ if: steps.check.outputs.should_build == 'true' && inputs.dry_run == false
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USER }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Resolve image config
+ if: steps.check.outputs.should_build == 'true'
+ id: config
+ run: |
+ component="${{ matrix.component }}"
+ case "$component" in
+ rust-server)
+ echo "image=apache/iggy" >> "$GITHUB_OUTPUT"
+ echo "dockerfile=core/server/Dockerfile" >> "$GITHUB_OUTPUT"
+ echo "binary=iggy-server" >> "$GITHUB_OUTPUT"
+ echo "prebuilt_arg=PREBUILT_IGGY_SERVER" >> "$GITHUB_OUTPUT"
+ echo "extra_args=--build-arg PREBUILT_IGGY_CLI=artifacts/iggy" >> "$GITHUB_OUTPUT"
+ ;;
+ rust-mcp)
+ echo "image=apache/iggy-mcp" >> "$GITHUB_OUTPUT"
+ echo "dockerfile=core/ai/mcp/Dockerfile" >> "$GITHUB_OUTPUT"
+ echo "binary=iggy-mcp" >> "$GITHUB_OUTPUT"
+ echo "prebuilt_arg=PREBUILT_IGGY_MCP" >> "$GITHUB_OUTPUT"
+ echo "extra_args=" >> "$GITHUB_OUTPUT"
+ ;;
+ rust-connectors)
+ echo "image=apache/iggy-connect" >> "$GITHUB_OUTPUT"
+ echo "dockerfile=core/connectors/runtime/Dockerfile" >> "$GITHUB_OUTPUT"
+ echo "binary=iggy-connectors" >> "$GITHUB_OUTPUT"
+ echo "prebuilt_arg=PREBUILT_IGGY_CONNECTORS" >> "$GITHUB_OUTPUT"
+ echo "extra_args=" >> "$GITHUB_OUTPUT"
+ ;;
+ rust-bench-dashboard)
+ echo "image=apache/iggy-bench-dashboard" >> "$GITHUB_OUTPUT"
+ echo "dockerfile=core/bench/dashboard/server/Dockerfile" >> "$GITHUB_OUTPUT"
+ echo "binary=iggy-bench-dashboard-server" >> "$GITHUB_OUTPUT"
+ echo "prebuilt_arg=PREBUILT_IGGY_BENCH_DASHBOARD_SERVER" >> "$GITHUB_OUTPUT"
+ echo "extra_args=--build-arg PREBUILT_FRONTEND_DIST=artifacts/dist" >> "$GITHUB_OUTPUT"
+ ;;
+ *)
+ echo "Unknown component: $component"
+ exit 1
+ ;;
+ esac
+
+ - name: Resolve version
+ if: steps.check.outputs.should_build == 'true'
+ id: version
+ run: |
+ input_version="${{ inputs.version }}"
+ component="${{ matrix.component }}"
+
+ # If version is "auto", extract actual version from component
+ if [ "$input_version" = "auto" ]; then
+ chmod +x scripts/extract-version.sh 2>/dev/null || true
+ VERSION=$(scripts/extract-version.sh "$component" 2>/dev/null || echo "")
+ if [ -z "$VERSION" ]; then
+ echo "❌ Failed to extract version for $component"
+ exit 1
+ fi
+ echo "📦 Extracted version for $component: $VERSION"
+ else
+ # Use provided version (e.g., "edge", "0.8.0")
+ VERSION="$input_version"
+ echo "📦 Using provided version: $VERSION"
+ fi
+
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+ - name: Build Docker image
+ if: steps.check.outputs.should_build == 'true'
+ run: |
+ IMAGE="${{ steps.config.outputs.image }}"
+ VERSION="${{ steps.version.outputs.version }}"
+ ARCH="${{ matrix.arch }}"
+ DOCKERFILE="${{ steps.config.outputs.dockerfile }}"
+ BINARY="${{ steps.config.outputs.binary }}"
+ PREBUILT_ARG="${{ steps.config.outputs.prebuilt_arg }}"
+ EXTRA_ARGS="${{ steps.config.outputs.extra_args }}"
+
+ TAG="${IMAGE}:${VERSION}-${ARCH}"
+
+ echo "Building $TAG"
+ echo " Dockerfile: $DOCKERFILE"
+ echo " Binary: artifacts/$BINARY"
+ echo " Prebuilt arg: $PREBUILT_ARG"
+
+ docker build \
+ --file "$DOCKERFILE" \
+ --target runtime-prebuilt \
+ --tag "$TAG" \
+ --build-arg "${PREBUILT_ARG}=artifacts/${BINARY}" \
+ $EXTRA_ARGS \
+ .
+
+ echo "Built image: $TAG"
+ docker images "$IMAGE"
+
+ - name: Push Docker image
+ if: steps.check.outputs.should_build == 'true' && inputs.dry_run == false
+ run: |
+ IMAGE="${{ steps.config.outputs.image }}"
+ VERSION="${{ steps.version.outputs.version }}"
+ ARCH="${{ matrix.arch }}"
+ TAG="${IMAGE}:${VERSION}-${ARCH}"
+
+ echo "Pushing $TAG"
+ docker push "$TAG"
+
+ - name: Save image digest
+ if: steps.check.outputs.should_build == 'true'
+ id: digest
+ run: |
+ IMAGE="${{ steps.config.outputs.image }}"
+ VERSION="${{ steps.version.outputs.version }}"
+ ARCH="${{ matrix.arch }}"
+ TAG="${IMAGE}:${VERSION}-${ARCH}"
+
+ if [ "${{ inputs.dry_run }}" = "false" ]; then
+ DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$TAG" 2>/dev/null || echo "")
+ echo "digest=$DIGEST" >> "$GITHUB_OUTPUT"
+ else
+ echo "digest=dry-run" >> "$GITHUB_OUTPUT"
+ fi
+
+ create-manifests:
+ name: Create manifests
+ needs: build-images
+ if: inputs.dry_run == false
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ component: [rust-server, rust-mcp, rust-connectors, rust-bench-dashboard]
+ steps:
+ - name: Check if component should be built
+ id: check
+ run: |
+ components="${{ inputs.components }}"
+ component="${{ matrix.component }}"
+ # Use word boundary matching to avoid partial matches
+ if echo "$components" | grep -wq "$component"; then
+ echo "should_build=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "should_build=false" >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Checkout code
+ if: steps.check.outputs.should_build == 'true'
+ uses: actions/checkout@v4
+
+ - name: Login to Docker Hub
+ if: steps.check.outputs.should_build == 'true'
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USER }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Resolve image name
+ if: steps.check.outputs.should_build == 'true'
+ id: config
+ run: |
+ component="${{ matrix.component }}"
+ case "$component" in
+ rust-server) echo "image=apache/iggy" >> "$GITHUB_OUTPUT" ;;
+ rust-mcp) echo "image=apache/iggy-mcp" >> "$GITHUB_OUTPUT" ;;
+ rust-connectors) echo "image=apache/iggy-connect" >> "$GITHUB_OUTPUT" ;;
+ rust-bench-dashboard) echo "image=apache/iggy-bench-dashboard" >> "$GITHUB_OUTPUT" ;;
+ esac
+
+ - name: Resolve version
+ if: steps.check.outputs.should_build == 'true'
+ id: version
+ run: |
+ input_version="${{ inputs.version }}"
+ component="${{ matrix.component }}"
+
+ # If version is "auto", extract actual version from component
+ if [ "$input_version" = "auto" ]; then
+ chmod +x scripts/extract-version.sh 2>/dev/null || true
+ VERSION=$(scripts/extract-version.sh "$component" 2>/dev/null || echo "")
+ if [ -z "$VERSION" ]; then
+ echo "❌ Failed to extract version for $component"
+ exit 1
+ fi
+ echo "📦 Extracted version for $component: $VERSION"
+ else
+ VERSION="$input_version"
+ echo "📦 Using provided version: $VERSION"
+ fi
+
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+ - name: Create and push manifest
+ if: steps.check.outputs.should_build == 'true'
+ run: |
+ IMAGE="${{ steps.config.outputs.image }}"
+ VERSION="${{ steps.version.outputs.version }}"
+
+ echo "Creating manifest for $IMAGE:$VERSION"
+
+ # Create manifest from platform-specific images
+ docker manifest create "${IMAGE}:${VERSION}" \
+ "${IMAGE}:${VERSION}-amd64" \
+ "${IMAGE}:${VERSION}-arm64"
+
+ # Push the manifest
+ docker manifest push "${IMAGE}:${VERSION}"
+
+ echo "Pushed manifest: ${IMAGE}:${VERSION}"
+
+ - name: Create and push latest tag
+ if: steps.check.outputs.should_build == 'true'
+ run: |
+ IMAGE="${{ steps.config.outputs.image }}"
+ VERSION="${{ steps.version.outputs.version }}"
+
+ # Only create 'latest' tag for stable releases (not edge, rc, alpha, beta)
+ if [ "$VERSION" = "edge" ]; then
+ echo "Skipping latest tag for edge version"
+ exit 0
+ fi
+
+ if echo "$VERSION" | grep -qE '(rc|alpha|beta)'; then
+ echo "Skipping latest tag for pre-release version: $VERSION"
+ exit 0
+ fi
+
+ echo "Creating latest manifest for $IMAGE (version: $VERSION)"
+
+ # Create manifest for latest
+ docker manifest create "${IMAGE}:latest" \
+ "${IMAGE}:${VERSION}-amd64" \
+ "${IMAGE}:${VERSION}-arm64"
+
+ # Push the latest manifest
+ docker manifest push "${IMAGE}:latest"
+
+ echo "Pushed manifest: ${IMAGE}:latest"
+
+ summary:
+ name: Summary
+ needs: [build-images, create-manifests]
+ if: always()
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Generate summary
+ run: |
+ {
+ echo "## Docker Images Build Summary"
+ echo ""
+ echo "| Component | Version | Status |"
+ echo "|-----------|---------|--------|"
+
+ components="${{ inputs.components }}"
+ input_version="${{ inputs.version }}"
+ dry_run="${{ inputs.dry_run }}"
+
+ for component in rust-server rust-mcp rust-connectors rust-bench-dashboard; do
+ # Use word boundary matching to avoid partial matches
+ if echo "$components" | grep -wq "$component"; then
+ case "$component" in
+ rust-server) image="apache/iggy" ;;
+ rust-mcp) image="apache/iggy-mcp" ;;
+ rust-connectors) image="apache/iggy-connect" ;;
+ rust-bench-dashboard) image="apache/iggy-bench-dashboard" ;;
+ esac
+
+ # Resolve version for display
+ if [ "$input_version" = "auto" ]; then
+ chmod +x scripts/extract-version.sh 2>/dev/null || true
+ version=$(scripts/extract-version.sh "$component" 2>/dev/null || echo "unknown")
+ else
+ version="$input_version"
+ fi
+
+ if [ "$dry_run" = "true" ]; then
+ echo "| $component | $version | Built (dry-run) |"
+ else
+ echo "| $component | $version | Published to \`$image:$version\` |"
+ fi
+ else
+ echo "| $component | - | Skipped |"
+ fi
+ done
+
+ echo ""
+ if [ "$dry_run" = "true" ]; then
+ echo "**Mode:** Dry run (images not pushed)"
+ else
+ echo "**Mode:** Production (images pushed to Docker Hub)"
+ fi
+ } >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/workflows/_build_rust_artifacts.yml b/.github/workflows/_build_rust_artifacts.yml
index f5b3c5006..8ecfe440a 100644
--- a/.github/workflows/_build_rust_artifacts.yml
+++ b/.github/workflows/_build_rust_artifacts.yml
@@ -41,7 +41,7 @@ on:
binaries:
type: string
required: false
- default: "iggy-server,iggy,iggy-bench,iggy-connectors"
+ default: "iggy-server,iggy,iggy-bench,iggy-connectors,iggy-mcp"
description: "Comma-separated list of binaries to build"
connector_plugins:
type: string
@@ -245,9 +245,102 @@ jobs:
retention-days: 7
if-no-files-found: error
+ build-bench-dashboard:
+ name: Bench dashboard ${{ matrix.target }}
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - target: x86_64-unknown-linux-gnu
+ runner: ubuntu-latest
+ - target: aarch64-unknown-linux-gnu
+ runner: ubuntu-24.04-arm
+ steps:
+ - name: Download latest copy script from master
+ if: inputs.use_latest_ci
+ run: |
+ curl -sSL "https://raw.githubusercontent.com/${{ github.repository }}/master/scripts/copy-latest-from-master.sh" \
+ -o /tmp/copy-latest-from-master.sh
+ chmod +x /tmp/copy-latest-from-master.sh
+
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.commit || github.sha }}
+
+ - name: Apply latest CI from master
+ if: inputs.use_latest_ci
+ run: |
+ /tmp/copy-latest-from-master.sh save .github scripts
+ /tmp/copy-latest-from-master.sh apply
+
+ - name: Setup Rust with cache
+ uses: ./.github/actions/utils/setup-rust-with-cache
+ with:
+ cache-targets: false
+
+ - name: Add Rust targets
+ run: |
+ rustup target add ${{ matrix.target }}
+ rustup target add wasm32-unknown-unknown
+
+ - name: Install trunk
+ run: |
+ # Install trunk for WASM frontend build
+ TRUNK_VERSION="0.21.15"
+ ARCH="$(uname -m)"
+ case "$ARCH" in
+ x86_64) TRUNK_ARCH="x86_64-unknown-linux-gnu" ;;
+ aarch64) TRUNK_ARCH="aarch64-unknown-linux-gnu" ;;
+ *)
+ echo "Unsupported architecture: $ARCH"
+ exit 1
+ ;;
+ esac
+ curl -L "https://github.com/trunk-rs/trunk/releases/download/v${TRUNK_VERSION}/trunk-${TRUNK_ARCH}.tar.gz" | tar xz
+ sudo mv trunk /usr/local/bin/
+ trunk --version
+
+ - name: Build frontend (WASM)
+ run: |
+ cd core/bench/dashboard/frontend
+ trunk build --release
+
+ - name: Build backend binary
+ run: |
+ cargo build --locked --release --target ${{ matrix.target }} --bin iggy-bench-dashboard-server
+
+ - name: Package bench-dashboard
+ id: pkg
+ run: |
+ target="${{ matrix.target }}"
+ version="${{ inputs.version }}"
+ outdir="dist/${target}"
+ mkdir -p "${outdir}/dist"
+
+ # Copy backend binary
+ cp "target/${target}/release/iggy-bench-dashboard-server" "${outdir}/"
+
+ # Copy frontend dist folder
+ cp -r core/bench/dashboard/frontend/dist/* "${outdir}/dist/"
+
+ tarball="iggy-bench-dashboard-${target}-${version}.tar.gz"
+ tar czf "${tarball}" -C "${outdir}" .
+ echo "tarball=${tarball}" >> "$GITHUB_OUTPUT"
+
+ - name: Upload artifact
+ if: inputs.upload_artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: bench-dashboard-${{ matrix.target }}
+ path: ${{ steps.pkg.outputs.tarball }}
+ retention-days: 7
+ if-no-files-found: error
+
collect:
name: Collect artifacts
- needs: [build-binaries, build-connector-plugins]
+ needs: [build-binaries, build-connector-plugins, build-bench-dashboard]
if: inputs.upload_artifacts
runs-on: ubuntu-latest
outputs:
@@ -288,6 +381,7 @@ jobs:
size=$(ls -lh "$tarball" | awk '{print $5}')
if [[ "$filename" == iggy-connectors-* ]]; then type="plugins"
+ elif [[ "$filename" == iggy-bench-dashboard-* ]]; then type="dashboard"
else type="binaries"; fi
if [[ "$filename" == *"x86_64"* ]]; then target="x86_64"
diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml
index 84a2bc9ed..50300cdb0 100644
--- a/.github/workflows/post-merge.yml
+++ b/.github/workflows/post-merge.yml
@@ -34,67 +34,6 @@ env:
IGGY_CI_BUILD: true
jobs:
- plan:
- name: Plan dockerhub components
- runs-on: ubuntu-latest
- outputs:
- matrix: ${{ steps.mk.outputs.matrix }}
- steps:
- - uses: actions/checkout@v4
-
- - name: Load publish config (base64)
- id: cfg
- shell: bash
- run: |
- if ! command -v yq >/dev/null 2>&1; then
- YQ_VERSION="v4.47.1"
- YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb"
- curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64
- echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1
- chmod +x /usr/local/bin/yq
- fi
- echo "components_b64=$(yq -o=json -I=0 '.components' .github/config/publish.yml | base64 -w0)" >> "$GITHUB_OUTPUT"
-
- - name: Build matrix
- id: mk
- uses: actions/github-script@v7
- with:
- script: |
- const b64 = `${{ steps.cfg.outputs.components_b64 }}` || '';
- if (!b64) {
- core.setOutput('matrix', JSON.stringify({ include: [{ component: 'noop' }] }));
- return;
- }
- const comps = JSON.parse(Buffer.from(b64, 'base64').toString('utf8'));
- const include = Object.entries(comps)
- .filter(([_, v]) => v && v.registry === 'dockerhub')
- .map(([k]) => ({ component: k }));
- const uniq = Array.from(new Map(include.map(i => [i.component, i])).values());
- core.setOutput('matrix', JSON.stringify(uniq.length ? { include: uniq } : { include: [{ component: 'noop' }] }));
-
- docker-edge:
- name: ${{ matrix.component }}
- needs: plan
- if: ${{ fromJson(needs.plan.outputs.matrix).include[0].component != 'noop' }}
- runs-on: ubuntu-latest
- timeout-minutes: 120
- strategy:
- fail-fast: false
- matrix: ${{ fromJson(needs.plan.outputs.matrix) }}
- env:
- DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
- steps:
- - uses: actions/checkout@v4
-
- - uses: ./.github/actions/utils/docker-buildx
- with:
- task: publish
- libc: musl
- component: ${{ matrix.component }}
- version: edge
- dry_run: ${{ github.event.repository.fork }} # forks: always dry-run
-
build-artifacts:
name: Build artifacts
uses: ./.github/workflows/_build_rust_artifacts.yml
@@ -102,6 +41,17 @@ jobs:
version: edge
upload_artifacts: true
+ docker-images:
+ name: Docker images
+ needs: build-artifacts
+ if: needs.build-artifacts.result == 'success'
+ uses: ./.github/workflows/_build_docker_images.yml
+ with:
+ version: edge
+ dry_run: ${{ github.event.repository.fork }}
+ components: "rust-server,rust-mcp,rust-connectors,rust-bench-dashboard"
+ secrets: inherit
+
create-prerelease:
name: Create edge pre-release
runs-on: ubuntu-latest
@@ -153,6 +103,10 @@ jobs:
- `iggy` - The command-line interface
- `iggy-bench` - The benchmarking tool
- `iggy-connectors` - The connectors runtime
+ - `iggy-mcp` - Model Context Protocol server for LLM integration
+
+ ## Bench Dashboard
+ - `iggy-bench-dashboard-server` - Benchmarking web dashboard (with frontend)
## Connector plugins included (.so)
- `iggy_connector_elasticsearch_sink`
@@ -176,6 +130,10 @@ jobs:
- `iggy-connectors-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64 (glibc)
- `iggy-connectors-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64 (glibc)
+ ### Bench Dashboard
+ - `iggy-bench-dashboard-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64 (glibc)
+ - `iggy-bench-dashboard-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64 (glibc)
+
## Build info
- Server version: ${{ steps.meta.outputs.server_version }}
- Commit: ${{ github.sha }}
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 77470d3c5..29856b00b 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -136,10 +136,13 @@ jobs:
outputs:
targets: ${{ steps.mk.outputs.targets }}
non_rust_targets: ${{ steps.mk.outputs.non_rust_targets }}
+ non_docker_targets: ${{ steps.mk.outputs.non_docker_targets }}
count: ${{ steps.mk.outputs.count }}
go_sdk_version: ${{ steps.mk.outputs.go_sdk_version }}
has_python: ${{ steps.mk.outputs.has_python }}
has_rust_crates: ${{ steps.mk.outputs.has_rust_crates }}
+ rust_docker_components: ${{ steps.mk.outputs.rust_docker_components }}
+ has_web_ui: ${{ steps.mk.outputs.has_web_ui }}
steps:
- name: Download latest copy script from master
if: inputs.use_latest_ci
@@ -263,9 +266,13 @@ jobs:
}
}
+ // Filter non-Docker targets (for SDK publishing, not handled by _build_docker_images.yml)
+ const nonDockerTargets = nonRustTargets.filter(t => t.type !== 'docker');
+
console.log(`Publishing ${targets.length} components:`);
targets.forEach(t => console.log(` - ${t.name} (${t.type}) -> ${t.registry || 'N/A'}`));
console.log(` (${nonRustTargets.length} non-Rust, ${targets.length - nonRustTargets.length} Rust crates)`);
+ console.log(` (${nonDockerTargets.length} non-Docker SDKs for matrix job)`);
// Output all targets for reference and tag creation
core.setOutput('targets', JSON.stringify(targets.length ? { include: targets } : { include: [{ key: 'noop', type: 'noop' }] }));
@@ -273,10 +280,24 @@ jobs:
// Output only non-Rust targets for the parallel publish job
core.setOutput('non_rust_targets', JSON.stringify(nonRustTargets.length ? { include: nonRustTargets } : { include: [{ key: 'noop', type: 'noop' }] }));
+ // Output only non-Docker, non-Rust targets (SDKs only)
+ core.setOutput('non_docker_targets', JSON.stringify(nonDockerTargets.length ? { include: nonDockerTargets } : { include: [{ key: 'noop', type: 'noop' }] }));
+
core.setOutput('count', String(targets.length));
core.setOutput('go_sdk_version', goVersion);
core.setOutput('has_rust_crates', String(hasRustCrates));
+ // Separate Rust Docker components from web-ui
+ // web-ui uses docker-buildx action directly (Node.js, doesn't need pre-built artifacts)
+ const rustDockerComponents = targets
+ .filter(t => t.type === 'docker' && t.key !== 'web-ui')
+ .map(t => t.key)
+ .join(',');
+ const hasWebUi = targets.some(t => t.key === 'web-ui');
+
+ core.setOutput('rust_docker_components', rustDockerComponents);
+ core.setOutput('has_web_ui', String(hasWebUi));
+
// Check if Python SDK is in targets and extract version
const pythonTarget = targets.find(t => t.key === 'sdk-python');
if (pythonTarget) {
@@ -455,6 +476,84 @@ jobs:
use_latest_ci: ${{ inputs.use_latest_ci }}
commit: ${{ needs.validate.outputs.commit }}
+ build-docker-artifacts:
+ name: Build Docker artifacts
+ needs: [validate, plan, check-tags]
+ if: |
+ needs.validate.outputs.has_targets == 'true' &&
+ inputs.publish_dockerhub != ''
+ uses: ./.github/workflows/_build_rust_artifacts.yml
+ with:
+ version: "publish"
+ upload_artifacts: true
+ use_latest_ci: ${{ inputs.use_latest_ci }}
+ commit: ${{ needs.validate.outputs.commit }}
+
+ publish-docker-images:
+ name: Publish Rust Docker images
+ needs: [validate, plan, check-tags, build-docker-artifacts]
+ if: |
+ needs.validate.outputs.has_targets == 'true' &&
+ needs.plan.outputs.rust_docker_components != '' &&
+ needs.build-docker-artifacts.result == 'success'
+ uses: ./.github/workflows/_build_docker_images.yml
+ with:
+ version: "auto"
+ dry_run: ${{ inputs.dry_run }}
+ components: ${{ needs.plan.outputs.rust_docker_components }}
+ secrets: inherit
+
+ # web-ui uses docker-buildx directly (Node.js, doesn't benefit from pre-built artifacts)
+ publish-web-ui:
+ name: Publish web-ui Docker image
+ needs: [validate, plan, check-tags]
+ if: |
+ needs.validate.outputs.has_targets == 'true' &&
+ needs.plan.outputs.has_web_ui == 'true'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Download latest copy script from master
+ if: inputs.use_latest_ci
+ run: |
+ curl -sSL "https://raw.githubusercontent.com/${{ github.repository }}/master/scripts/copy-latest-from-master.sh" \
+ -o /tmp/copy-latest-from-master.sh
+ chmod +x /tmp/copy-latest-from-master.sh
+ echo "✅ Downloaded latest copy script from master"
+
+ - name: Checkout at commit
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ needs.validate.outputs.commit }}
+
+ - name: Save and apply latest CI from master
+ if: inputs.use_latest_ci
+ run: |
+ /tmp/copy-latest-from-master.sh save \
+ .github \
+ scripts \
+ web/Dockerfile
+
+ /tmp/copy-latest-from-master.sh apply
+
+ - name: Extract web-ui version
+ id: version
+ run: |
+ chmod +x scripts/extract-version.sh
+ VERSION=$(scripts/extract-version.sh web-ui)
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+ echo "📦 web-ui version: $VERSION"
+
+ - name: Build and push web-ui image
+ uses: ./.github/actions/utils/docker-buildx
+ env:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ with:
+ task: publish
+ component: web-ui
+ version: ${{ steps.version.outputs.version }}
+ dry_run: ${{ inputs.dry_run }}
+
# Sequential Rust crate publishing to handle dependencies properly
publish-rust-crates:
name: Publish Rust Crates
@@ -610,17 +709,15 @@ jobs:
if: |
always() &&
needs.validate.outputs.has_targets == 'true' &&
- fromJson(needs.plan.outputs.non_rust_targets).include[0].key != 'noop' &&
+ fromJson(needs.plan.outputs.non_docker_targets).include[0].key != 'noop' &&
(needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped') &&
(needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped')
runs-on: ubuntu-latest
strategy:
fail-fast: false
- matrix: ${{ fromJson(needs.plan.outputs.non_rust_targets) }}
+ matrix: ${{ fromJson(needs.plan.outputs.non_docker_targets) }}
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
- DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
NEXUS_USER: ${{ secrets.NEXUS_USER }}
@@ -667,7 +764,7 @@ jobs:
test -x scripts/extract-version.sh || chmod +x scripts/extract-version.sh
- name: Setup Rust toolchain (if needed)
- if: matrix.type == 'rust' || matrix.type == 'docker' || matrix.type == 'python'
+ if: matrix.type == 'rust' || matrix.type == 'python'
uses: ./.github/actions/utils/setup-rust-with-cache
with:
cache-targets: false
@@ -696,18 +793,6 @@ jobs:
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
echo "✅ Resolved ${{ matrix.key }} -> version=$VERSION tag=${TAG:-<none>}"
- # ─────────────────────────────────────────
- # Docker Publishing
- # ─────────────────────────────────────────
- - name: Publish Docker image
- if: matrix.type == 'docker'
- uses: ./.github/actions/utils/docker-buildx
- with:
- task: publish
- component: ${{ matrix.key }}
- version: ${{ steps.ver.outputs.version }}
- dry_run: ${{ inputs.dry_run }}
-
# ─────────────────────────────────────────
# Python SDK Publishing
# ─────────────────────────────────────────
@@ -774,6 +859,8 @@ jobs:
check-tags,
build-python-wheels,
publish-rust-crates,
+ publish-docker-images,
+ publish-web-ui,
publish,
]
if: |
@@ -782,6 +869,8 @@ jobs:
inputs.dry_run == false &&
inputs.skip_tag_creation == false &&
(needs.publish.result == 'success' || needs.publish.result == 'skipped') &&
+ (needs.publish-docker-images.result == 'success' || needs.publish-docker-images.result == 'skipped') &&
+ (needs.publish-web-ui.result == 'success' || needs.publish-web-ui.result == 'skipped') &&
(needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped') &&
(needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped')
runs-on: ubuntu-latest
@@ -873,6 +962,8 @@ jobs:
check-tags,
build-python-wheels,
publish-rust-crates,
+ publish-docker-images,
+ publish-web-ui,
publish,
create-tags,
]
@@ -1055,6 +1146,8 @@ jobs:
plan,
build-python-wheels,
publish-rust-crates,
+ publish-docker-images,
+ publish-web-ui,
publish,
create-tags,
summary,
diff --git a/core/ai/mcp/Dockerfile b/core/ai/mcp/Dockerfile
index f5fa7e977..6566d347d 100644
--- a/core/ai/mcp/Dockerfile
+++ b/core/ai/mcp/Dockerfile
@@ -85,7 +85,26 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG
fi
#
-# Final runtime - Debian trixie Slim
+# Prebuilt path (FAST) - for CI with pre-compiled binaries
+#
+FROM debian:trixie-slim AS prebuilt
+ARG PREBUILT_IGGY_MCP
+WORKDIR /out
+COPY ${PREBUILT_IGGY_MCP} /out/iggy-mcp
+RUN chmod +x /out/iggy-mcp
+
+#
+# Final runtime using prebuilt binary
+#
+FROM debian:trixie-slim AS runtime-prebuilt
+WORKDIR /app
+
+COPY --from=prebuilt /out/iggy-mcp /usr/local/bin/iggy-mcp
+
+ENTRYPOINT ["iggy-mcp"]
+
+#
+# Final runtime - Debian trixie Slim (from source build)
#
FROM debian:trixie-slim AS runtime
WORKDIR /app
diff --git a/core/bench/dashboard/server/Dockerfile b/core/bench/dashboard/server/Dockerfile
index 29e103824..417e1ff48 100644
--- a/core/bench/dashboard/server/Dockerfile
+++ b/core/bench/dashboard/server/Dockerfile
@@ -53,8 +53,66 @@ RUN cd core/bench/dashboard/frontend && trunk build --release
# Build the server with release profile
RUN cargo build --release --bin iggy-bench-dashboard-server
-# Runtime stage
-FROM debian:trixie-slim
+#
+# Prebuilt path (FAST) - for CI with pre-compiled binaries and frontend
+#
+FROM debian:trixie-slim AS prebuilt
+ARG PREBUILT_IGGY_BENCH_DASHBOARD_SERVER
+ARG PREBUILT_FRONTEND_DIST
+WORKDIR /out
+COPY ${PREBUILT_IGGY_BENCH_DASHBOARD_SERVER} /out/iggy-bench-dashboard-server
+COPY ${PREBUILT_FRONTEND_DIST} /out/frontend/dist
+RUN chmod +x /out/iggy-bench-dashboard-server
+
+#
+# Runtime stage using prebuilt binaries
+#
+FROM debian:trixie-slim AS runtime-prebuilt
+
+WORKDIR /app
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ ca-certificates \
+ openssl \
+ curl \
+ libssl3 \
+ && rm -rf /var/lib/apt/lists/*
+
+# Copy prebuilt binary and frontend files
+COPY --from=prebuilt /out/iggy-bench-dashboard-server /app/
+COPY --from=prebuilt /out/frontend/dist /app/frontend/dist
+
+# Create data directory and non-root user
+RUN groupadd -r iggy && \
+ useradd -r -g iggy -s /bin/false iggy && \
+ mkdir -p /data/performance_results && \
+ chown -R iggy:iggy /app /data && \
+ chmod -R 755 /data/performance_results
+
+# Copy the entrypoint script
+COPY core/bench/dashboard/server/docker-entrypoint.sh /app/
+RUN chmod +x /app/docker-entrypoint.sh && \
+ chown iggy:iggy /app/docker-entrypoint.sh
+
+# Set default environment variables for configuration
+ENV HOST=0.0.0.0 \
+ PORT=80 \
+ RESULTS_DIR=/data/performance_results
+
+# Set volume for results with proper permissions
+VOLUME ["/data/performance_results"]
+
+# Switch to non-root user
+USER iggy
+
+# Set the entrypoint script
+ENTRYPOINT ["/app/docker-entrypoint.sh"]
+
+#
+# Runtime stage (from source build)
+#
+FROM debian:trixie-slim AS runtime
WORKDIR /app
diff --git a/core/connectors/runtime/Dockerfile b/core/connectors/runtime/Dockerfile
index 7e78fe1a5..c65159133 100644
--- a/core/connectors/runtime/Dockerfile
+++ b/core/connectors/runtime/Dockerfile
@@ -85,7 +85,26 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG
fi
#
-# Final runtime - Debian trixie Slim
+# Prebuilt path (FAST) - for CI with pre-compiled binaries
+#
+FROM debian:trixie-slim AS prebuilt
+ARG PREBUILT_IGGY_CONNECTORS
+WORKDIR /out
+COPY ${PREBUILT_IGGY_CONNECTORS} /out/iggy-connectors
+RUN chmod +x /out/iggy-connectors
+
+#
+# Final runtime using prebuilt binary
+#
+FROM debian:trixie-slim AS runtime-prebuilt
+WORKDIR /app
+
+COPY --from=prebuilt /out/iggy-connectors /usr/local/bin/iggy-connectors
+
+ENTRYPOINT ["iggy-connectors"]
+
+#
+# Final runtime - Debian trixie Slim (from source build)
#
FROM debian:trixie-slim AS runtime
WORKDIR /app