You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by le...@apache.org on 2026/04/26 18:43:09 UTC
(fluss-rust) branch main updated: feat: Basic CI/CD for elixir (#497)
This is an automated email from the ASF dual-hosted git repository.
leekeiabstraction pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 69941d5 feat: Basic CI/CD for elixir (#497)
69941d5 is described below
commit 69941d5d926009e18bca7bd0cdf602ad665d29cc
Author: Anton Borisov <14...@users.noreply.github.com>
AuthorDate: Sun Apr 26 20:43:04 2026 +0200
feat: Basic CI/CD for elixir (#497)
* [elixir] Basic CI/CD for elixir
* plain install approach
* use now allowlisted action
---
.github/workflows/build_and_test_cpp.yml | 1 +
...d_test_python.yml => build_and_test_elixir.yml} | 63 ++++--
.github/workflows/build_and_test_python.yml | 1 +
.github/workflows/build_and_test_rust.yml | 1 +
bindings/elixir/test/support/cluster.ex | 229 +++++++--------------
5 files changed, 119 insertions(+), 176 deletions(-)
diff --git a/.github/workflows/build_and_test_cpp.yml b/.github/workflows/build_and_test_cpp.yml
index c411076..9ede0c3 100644
--- a/.github/workflows/build_and_test_cpp.yml
+++ b/.github/workflows/build_and_test_cpp.yml
@@ -31,6 +31,7 @@ on:
- 'website/**'
- '**/*.md'
- 'bindings/python/**'
+ - 'bindings/elixir/**'
workflow_dispatch:
concurrency:
diff --git a/.github/workflows/build_and_test_python.yml b/.github/workflows/build_and_test_elixir.yml
similarity index 52%
copy from .github/workflows/build_and_test_python.yml
copy to .github/workflows/build_and_test_elixir.yml
index eb392d1..03f3329 100644
--- a/.github/workflows/build_and_test_python.yml
+++ b/.github/workflows/build_and_test_elixir.yml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-name: Python Build and Tests
+name: Elixir Build and Tests
on:
push:
@@ -31,6 +31,7 @@ on:
- 'website/**'
- '**/*.md'
- 'bindings/cpp/**'
+ - 'bindings/python/**'
workflow_dispatch:
concurrency:
@@ -41,42 +42,62 @@ jobs:
build-and-test:
timeout-minutes: 60
runs-on: ubuntu-latest
- strategy:
- matrix:
- python: ["3.9", "3.10", "3.11", "3.12"]
env:
+ OTP_VERSION: "28.0.2"
+ ELIXIR_VERSION: "1.19.5"
FLUSS_TEST_CLUSTER_BIN: ${{ github.workspace }}/target/debug/fluss-test-cluster
+ MIX_ENV: test
steps:
- uses: actions/checkout@v6
- - name: Set up Python
- uses: actions/setup-python@v6
+ - name: Set up BEAM
+ uses: erlef/setup-beam@fc68ffb90438ef2936bbb3251622353b3dcb2f93 # v1.24.0
with:
- python-version: ${{ matrix.python }}
-
- - name: Install uv
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78
+ otp-version: ${{ env.OTP_VERSION }}
+ elixir-version: ${{ env.ELIXIR_VERSION }}
- name: Install protoc
- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
- with:
- repo-token: ${{ secrets.GITHUB_TOKEN }}
+ run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Rust Cache
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+ - name: Cache Mix deps and build
+ uses: actions/cache@v4
+ with:
+ path: |
+ bindings/elixir/deps
+ bindings/elixir/_build
+ key: ${{ runner.os }}-mix-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-${{ hashFiles('bindings/elixir/mix.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-mix-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-
+
- name: Build fluss-test-cluster binary
run: cargo build -p fluss-test-cluster
- - name: Build Python bindings
- working-directory: bindings/python
- run: |
- uv sync --extra dev --no-install-project
- uv run --no-sync maturin develop --uv
+ - name: Fetch Elixir deps
+ working-directory: bindings/elixir
+ run: mix deps.get
+
+ - name: Check formatting
+ working-directory: bindings/elixir
+ run: mix format --check-formatted
+
+ - name: Compile (warnings as errors)
+ working-directory: bindings/elixir
+ run: mix compile --warnings-as-errors
+
+ - name: Credo
+ working-directory: bindings/elixir
+ run: mix credo
+
+ - name: Run unit tests
+ working-directory: bindings/elixir
+ run: mix test
- - name: Run tests (parallel)
- working-directory: bindings/python
- run: uv run --no-sync pytest test/ -v -n auto
+ - name: Run integration tests
+ working-directory: bindings/elixir
+ run: mix test --include integration --only integration
env:
RUST_LOG: DEBUG
RUST_BACKTRACE: full
diff --git a/.github/workflows/build_and_test_python.yml b/.github/workflows/build_and_test_python.yml
index eb392d1..88c1838 100644
--- a/.github/workflows/build_and_test_python.yml
+++ b/.github/workflows/build_and_test_python.yml
@@ -31,6 +31,7 @@ on:
- 'website/**'
- '**/*.md'
- 'bindings/cpp/**'
+ - 'bindings/elixir/**'
workflow_dispatch:
concurrency:
diff --git a/.github/workflows/build_and_test_rust.yml b/.github/workflows/build_and_test_rust.yml
index 4615575..af10869 100644
--- a/.github/workflows/build_and_test_rust.yml
+++ b/.github/workflows/build_and_test_rust.yml
@@ -32,6 +32,7 @@ on:
- '**/*.md'
- 'bindings/python/**'
- 'bindings/cpp/**'
+ - 'bindings/elixir/**'
workflow_dispatch:
concurrency:
diff --git a/bindings/elixir/test/support/cluster.ex b/bindings/elixir/test/support/cluster.ex
index 59598b4..40f0f68 100644
--- a/bindings/elixir/test/support/cluster.ex
+++ b/bindings/elixir/test/support/cluster.ex
@@ -18,21 +18,11 @@
defmodule Fluss.Test.Cluster do
@moduledoc false
- @fluss_image "apache/fluss"
- @fluss_version "0.9.0-incubating"
+ # Shells out to the `fluss-test-cluster` CLI (from `crates/fluss-test-cluster`),
+ # the same binary used by the Python and C++ integration tests.
- @network_name "fluss-elixir-test-network"
- @zookeeper_name "zookeeper-elixir-test"
- @coordinator_name "coordinator-server-elixir-test"
- @tablet_server_name "tablet-server-elixir-test"
-
- # Same fixed ports used by Python/C++ integration tests.
- @coordinator_sasl_port 9123
- @coordinator_plain_port 9223
- @tablet_sasl_port 9124
- @tablet_plain_port 9224
-
- def bootstrap_servers, do: "127.0.0.1:#{@coordinator_plain_port}"
+ @cluster_name "shared-test"
+ @cluster_json_prefix "CLUSTER_JSON: "
def ensure_started do
case System.get_env("FLUSS_BOOTSTRAP_SERVERS") do
@@ -42,170 +32,99 @@ defmodule Fluss.Test.Cluster do
end
def stop do
- for name <- [@tablet_server_name, @coordinator_name, @zookeeper_name] do
- System.cmd("docker", ["rm", "-f", name], stderr_to_stdout: true)
- end
+ if System.get_env("FLUSS_BOOTSTRAP_SERVERS") do
+ :ok
+ else
+ case find_cli_binary() do
+ {:ok, cli} ->
+ System.cmd(cli, ["stop", "--name", @cluster_name], stderr_to_stdout: true)
+ :ok
- System.cmd("docker", ["network", "rm", @network_name], stderr_to_stdout: true)
- :ok
+ {:error, _} ->
+ :ok
+ end
+ end
end
defp start_cluster do
- if port_open?(@coordinator_plain_port) do
- IO.puts("Reusing existing Fluss cluster on port #{@coordinator_plain_port}")
- {:ok, bootstrap_servers()}
+ with {:ok, cli} <- find_cli_binary(),
+ {output, 0} <-
+ System.cmd(cli, ["start", "--sasl", "--name", @cluster_name], stderr_to_stdout: true),
+ {:ok, bootstrap} <- parse_cluster_json(output) do
+ {:ok, bootstrap}
else
- do_start_cluster()
+ {output, code} when is_binary(output) ->
+ {:error, "fluss-test-cluster start failed (exit #{code}):\n#{output}"}
+
+ {:error, _} = err ->
+ err
end
end
- defp do_start_cluster do
- IO.puts("Starting Fluss cluster via Docker...")
+ defp find_cli_binary do
+ case System.get_env("FLUSS_TEST_CLUSTER_BIN") do
+ bin when is_binary(bin) and bin != "" ->
+ if File.regular?(bin),
+ do: {:ok, bin},
+ else: {:error, "FLUSS_TEST_CLUSTER_BIN=#{bin} does not exist"}
- # Remove any leftover containers from previous runs
- for name <- [@tablet_server_name, @coordinator_name, @zookeeper_name] do
- System.cmd("docker", ["rm", "-f", name], stderr_to_stdout: true)
- end
-
- System.cmd("docker", ["network", "create", @network_name], stderr_to_stdout: true)
-
- sasl_jaas =
- ~s(org.apache.fluss.security.auth.sasl.plain.PlainLoginModule required user_admin="admin-secret" user_alice="alice-secret";)
-
- coordinator_props =
- Enum.join(
- [
- "zookeeper.address: #{@zookeeper_name}:2181",
- "bind.listeners: INTERNAL://#{@coordinator_name}:0, CLIENT://#{@coordinator_name}:9123, PLAIN_CLIENT://#{@coordinator_name}:9223",
- "advertised.listeners: CLIENT://localhost:#{@coordinator_sasl_port}, PLAIN_CLIENT://localhost:#{@coordinator_plain_port}",
- "internal.listener.name: INTERNAL",
- "security.protocol.map: CLIENT:sasl",
- "security.sasl.enabled.mechanisms: plain",
- "security.sasl.plain.jaas.config: #{sasl_jaas}",
- "netty.server.num-network-threads: 1",
- "netty.server.num-worker-threads: 3"
- ],
- "\n"
- )
-
- tablet_props =
- Enum.join(
- [
- "zookeeper.address: #{@zookeeper_name}:2181",
- "bind.listeners: INTERNAL://#{@tablet_server_name}:0, CLIENT://#{@tablet_server_name}:9123, PLAIN_CLIENT://#{@tablet_server_name}:9223",
- "advertised.listeners: CLIENT://localhost:#{@tablet_sasl_port}, PLAIN_CLIENT://localhost:#{@tablet_plain_port}",
- "internal.listener.name: INTERNAL",
- "security.protocol.map: CLIENT:sasl",
- "security.sasl.enabled.mechanisms: plain",
- "security.sasl.plain.jaas.config: #{sasl_jaas}",
- "tablet-server.id: 0",
- "netty.server.num-network-threads: 1",
- "netty.server.num-worker-threads: 3"
- ],
- "\n"
- )
-
- docker_run([
- "--name",
- @zookeeper_name,
- "--network",
- @network_name,
- "-d",
- "zookeeper:3.9.2"
- ])
-
- docker_run([
- "--name",
- @coordinator_name,
- "--network",
- @network_name,
- "-p",
- "#{@coordinator_sasl_port}:9123",
- "-p",
- "#{@coordinator_plain_port}:9223",
- "-e",
- "FLUSS_PROPERTIES=#{coordinator_props}",
- "-d",
- "#{@fluss_image}:#{@fluss_version}",
- "coordinatorServer"
- ])
-
- docker_run([
- "--name",
- @tablet_server_name,
- "--network",
- @network_name,
- "-p",
- "#{@tablet_sasl_port}:9123",
- "-p",
- "#{@tablet_plain_port}:9223",
- "-e",
- "FLUSS_PROPERTIES=#{tablet_props}",
- "-d",
- "#{@fluss_image}:#{@fluss_version}",
- "tabletServer"
- ])
-
- all_ports = [@coordinator_plain_port, @tablet_plain_port]
-
- if wait_for_ports(all_ports, 90) do
- IO.puts("Fluss cluster started successfully.")
- {:ok, bootstrap_servers()}
- else
- {:error, "Cluster ports did not become ready within timeout"}
+ _ ->
+ locate_via_cargo()
end
end
- defp docker_run(args) do
- {output, code} = System.cmd("docker", ["run" | args], stderr_to_stdout: true)
+ defp locate_via_cargo do
+ case System.cmd("cargo", ["locate-project", "--workspace", "--message-format", "plain"],
+ stderr_to_stdout: true
+ ) do
+ {output, 0} ->
+ output |> String.trim() |> Path.dirname() |> find_binary_in_target()
- if code != 0 do
- IO.puts("Docker run warning (code #{code}): #{output}")
+ {output, code} ->
+ {:error, "cargo locate-project failed (exit #{code}): #{output}"}
end
end
- defp wait_for_ports(ports, timeout_s) do
- deadline = System.monotonic_time(:second) + timeout_s
+ defp find_binary_in_target(root) do
+ Enum.find_value(
+ ["debug", "release"],
+ {:error, "fluss-test-cluster binary not found. Run: cargo build -p fluss-test-cluster"},
+ &check_binary(root, &1)
+ )
+ end
- Enum.all?(ports, fn port ->
- remaining = deadline - System.monotonic_time(:second)
- remaining > 0 and wait_for_port(port, remaining)
- end)
+ defp check_binary(root, profile) do
+ path = Path.join([root, "target", profile, "fluss-test-cluster"])
+ if File.regular?(path), do: {:ok, path}, else: nil
end
- defp wait_for_port(port, timeout_s) do
- deadline = System.monotonic_time(:second) + timeout_s
+ defp parse_cluster_json(output) do
+ output
+ |> String.split("\n", trim: true)
+ |> Enum.find_value(
+ {:error, "No #{@cluster_json_prefix} token in output:\n#{output}"},
+ &extract_bootstrap/1
+ )
+ end
- Stream.repeatedly(fn ->
- case :gen_tcp.connect(~c"localhost", port, [], 1000) do
- {:ok, socket} ->
- :gen_tcp.close(socket)
- :ok
+ defp extract_bootstrap(line) do
+ case String.split(line, @cluster_json_prefix, parts: 2) do
+ [_, json] ->
+ case decode_bootstrap(json) do
+ {:ok, bootstrap} -> {:ok, bootstrap}
+ _ -> nil
+ end
- {:error, _} ->
- Process.sleep(1000)
- :retry
- end
- end)
- |> Enum.reduce_while(false, fn
- :ok, _acc ->
- {:halt, true}
-
- :retry, _acc ->
- if System.monotonic_time(:second) >= deadline,
- do: {:halt, false},
- else: {:cont, false}
- end)
+ _ ->
+ nil
+ end
end
- defp port_open?(port) do
- case :gen_tcp.connect(~c"localhost", port, [], 1000) do
- {:ok, socket} ->
- :gen_tcp.close(socket)
- true
-
- {:error, _} ->
- false
+ # Minimal JSON extractor for `bootstrap_servers`: avoids adding a JSON dep just for tests.
+ defp decode_bootstrap(json) do
+ case Regex.run(~r/"bootstrap_servers"\s*:\s*"([^"]+)"/, json) do
+ [_, servers] -> {:ok, servers}
+ _ -> {:error, "no bootstrap_servers in: #{json}"}
end
end
end