You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/07/27 16:05:40 UTC
(opendal-reqsign) 01/01: ci: migrate crate publishing to trusted publishing
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/trusted-publishing
in repository https://gitbox.apache.org/repos/asf/opendal-reqsign.git
commit 6a1b7c35ad0281d9a180bb0b5cdbeb37740ff8e1
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Jul 28 00:05:17 2026 +0800
ci: migrate crate publishing to trusted publishing
---
.asf.yaml | 29 +
.github/scripts/release_rust/README.md | 61 ++
.github/scripts/release_rust/bootstrap.py | 754 +++++++++++++++++++++
.github/scripts/release_rust/plan.py | 196 ++++++
.github/scripts/release_rust/publish.py | 142 ++++
.github/scripts/release_rust/test_bootstrap.py | 456 +++++++++++++
.github/scripts/release_rust/test_plan.py | 144 ++++
.github/scripts/release_rust/test_publish.py | 147 ++++
.../release_rust/test_trusted_publishing.py | 116 ++++
.github/scripts/release_rust/trusted_publishing.py | 162 +++++
.github/workflows/bootstrap_rust_crates.yml | 76 +++
.github/workflows/release.yml | 53 +-
.gitignore | 1 +
release/SKILL.md | 77 ++-
release/scripts/bootstrap-rust-crates.sh | 152 +++++
15 files changed, 2561 insertions(+), 5 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
index 308f2cf..20b5d5c 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -34,6 +34,35 @@ github:
issues: true
projects: false
discussions: true
+ environments:
+ release:
+ wait_timer: 0
+ deployment_branch_policy:
+ protected_branches: false
+ policies:
+ - name: "v*.*.*"
+ type: tag
+ rust-bootstrap:
+ required_reviewers:
+ - id: Xuanwo
+ type: User
+ - id: dentiny
+ type: User
+ - id: erickguan
+ type: User
+ - id: tisonkun
+ type: User
+ - id: koushiro
+ type: User
+ - id: PsiACE
+ type: User
+ wait_timer: 0
+ prevent_self_review: true
+ deployment_branch_policy:
+ protected_branches: false
+ policies:
+ - name: main
+ type: branch
custom_subjects:
new_discussion: "{title}"
edit_discussion: "Re: {title}"
diff --git a/.github/scripts/release_rust/README.md b/.github/scripts/release_rust/README.md
new file mode 100644
index 0000000..ab2aa66
--- /dev/null
+++ b/.github/scripts/release_rust/README.md
@@ -0,0 +1,61 @@
+# Rust release helpers
+
+These scripts define and publish the crates.io package set for Apache OpenDAL
+reqsign.
+
+## Publish plan
+
+`plan.py` reads `cargo metadata --no-deps`, selects workspace packages that can
+publish to crates.io, and orders them by non-dev local dependencies.
+
+```bash
+python3 .github/scripts/release_rust/plan.py
+```
+
+The release workflow first runs the complete workspace dry run:
+
+```bash
+cargo publish --workspace --dry-run
+```
+
+It then runs `publish.py`. The publisher uploads one package at a time, requests
+and revokes a fresh GitHub OIDC-derived crates.io token for every attempt,
+retries crates.io rate limits, and treats an already-published matching
+name/version as a completed step. Individual uploads use `--no-verify` because
+the workflow has already verified the complete package set from the same
+commit.
+
+Live publishing requires the `release` GitHub environment and a job with
+`id-token: write`. It never reads a long-lived crates.io token.
+
+## Bootstrap crate names
+
+Trusted Publishing cannot create the first version of a crate.
+`bootstrap.py` reserves new names and audits the complete publish plan:
+
+- `discover` reports missing names and existing `0.0.0` placeholders without
+ credentials.
+- `apply` authenticates every existing crate before any write, publishes a
+ dependency-free `0.0.0` namespace reservation for each missing name,
+ configures the exact `apache/opendal-reqsign`, `release.yml`, `release`
+ Trusted Publisher, enables `trustpub_only`, and performs a final authenticated
+ audit.
+- `verify` checks public crate metadata and `trustpub_only`. It cannot verify
+ ownership or the exact Trusted Publisher because those APIs require
+ authentication.
+
+The protected `rust-bootstrap` workflow always runs the authenticated audit,
+including when discovery finds no missing names. It never changes an
+established crate. Existing crates must be migrated independently before the
+workflow can succeed.
+
+Version `0.0.0` is an irreversible namespace reservation. It is not an ASF
+software release and contains no implementation.
+
+## Tests
+
+```bash
+python3 -m unittest discover \
+ -s .github/scripts/release_rust \
+ -p "test_*.py"
+```
diff --git a/.github/scripts/release_rust/bootstrap.py b/.github/scripts/release_rust/bootstrap.py
new file mode 100644
index 0000000..8c01322
--- /dev/null
+++ b/.github/scripts/release_rust/bootstrap.py
@@ -0,0 +1,754 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+
+from plan import PROJECT_DIR
+from plan import plan
+from publish import parse_retry_after
+from publish import should_retry
+
+
+REGISTRY_URL = "https://crates.io"
+REPOSITORY = "https://github.com/apache/opendal-reqsign"
+PLACEHOLDER_VERSION = "0.0.0"
+PLACEHOLDER_DESCRIPTION = (
+ "Namespace reservation for a crate planned by Apache OpenDAL reqsign."
+)
+PUBLISHER = {
+ "repository_owner": "apache",
+ "repository_name": "opendal-reqsign",
+ "workflow_filename": "release.yml",
+ "environment": "release",
+}
+USER_AGENT = (
+ "apache-opendal-reqsign-release-bootstrap/1.0 "
+ "(https://github.com/apache/opendal-reqsign)"
+)
+
+
+class ApiError(RuntimeError):
+ def __init__(self, operation: str, status: int, detail: str):
+ super().__init__(f"{operation} failed with HTTP {status}: {detail}")
+ self.status = status
+
+
+@dataclass(frozen=True)
+class PlannedCrate:
+ name: str
+ path: str
+
+
+@dataclass(frozen=True)
+class ReconcileResult:
+ name: str
+ actions: tuple[str, ...]
+
+
+def planned_crates(project_dir: Path = PROJECT_DIR) -> list[PlannedCrate]:
+ packages = [
+ PlannedCrate(name=package.name, path=package.path)
+ for package in plan(project_dir.resolve())
+ ]
+ names = [package.name for package in packages]
+ if len(names) != len(set(names)):
+ raise RuntimeError("duplicate crates.io package name in publish plan")
+ return packages
+
+
+def _api_error_detail(error: urllib.error.HTTPError) -> str:
+ try:
+ response = error.read().decode("utf-8", errors="replace")
+ finally:
+ error.close()
+ try:
+ body = json.loads(response)
+ errors = body.get("errors", [])
+ return "; ".join(item["detail"] for item in errors)
+ except (json.JSONDecodeError, AttributeError, KeyError, TypeError):
+ return response
+
+
+class CratesIoClient:
+ def __init__(self, registry_url: str = REGISTRY_URL, token: str | None = None):
+ self.registry_url = registry_url.rstrip("/")
+ self.token = token
+
+ def _request(
+ self,
+ method: str,
+ path: str,
+ operation: str,
+ body: dict[str, object] | None = None,
+ authenticated: bool = False,
+ ) -> dict[str, object]:
+ headers = {
+ "Accept": "application/json",
+ "User-Agent": USER_AGENT,
+ }
+ if authenticated:
+ if not self.token:
+ raise RuntimeError(f"{operation} requires a crates.io API token")
+ headers["Authorization"] = self.token
+
+ data = None
+ if body is not None:
+ data = json.dumps(body).encode()
+ headers["Content-Type"] = "application/json"
+
+ request = urllib.request.Request(
+ f"{self.registry_url}{path}",
+ data=data,
+ headers=headers,
+ method=method,
+ )
+ retryable = method in {"GET", "PATCH"}
+ for attempt in range(5):
+ try:
+ with urllib.request.urlopen(request, timeout=30) as response:
+ result = json.load(response)
+ if not isinstance(result, dict):
+ raise RuntimeError(f"{operation} returned an invalid response")
+ return result
+ except urllib.error.HTTPError as error:
+ retry = retryable and error.code in {429, 502, 503, 504}
+ if retry and attempt < 4:
+ retry_after = (
+ error.headers.get("Retry-After") if error.headers else None
+ )
+ delay = (
+ int(retry_after)
+ if retry_after and retry_after.isdigit()
+ else 2**attempt
+ )
+ error.read()
+ error.close()
+ print(
+ f"{operation} returned HTTP {error.code}; retrying in {delay}s",
+ flush=True,
+ )
+ time.sleep(delay)
+ continue
+ raise ApiError(
+ operation, error.code, _api_error_detail(error)
+ ) from None
+ except urllib.error.URLError as error:
+ if retryable and attempt < 4:
+ delay = 2**attempt
+ print(
+ f"{operation} failed: {error.reason}; retrying in {delay}s",
+ flush=True,
+ )
+ time.sleep(delay)
+ continue
+ raise RuntimeError(f"{operation} failed: {error.reason}") from None
+
+ raise AssertionError("unreachable")
+
+ def get_crate(self, name: str) -> dict[str, object] | None:
+ encoded_name = urllib.parse.quote(name, safe="")
+ try:
+ response = self._request(
+ "GET",
+ f"/api/v1/crates/{encoded_name}",
+ f"reading crate {name}",
+ )
+ except ApiError as error:
+ if error.status == 404:
+ return None
+ raise
+ krate = response.get("crate")
+ if not isinstance(krate, dict):
+ raise RuntimeError(f"crates.io returned invalid metadata for {name}")
+ return krate
+
+ def get_version(self, name: str, version: str) -> dict[str, object]:
+ encoded_name = urllib.parse.quote(name, safe="")
+ encoded_version = urllib.parse.quote(version, safe="")
+ response = self._request(
+ "GET",
+ f"/api/v1/crates/{encoded_name}/{encoded_version}",
+ f"reading {name} {version}",
+ )
+ version_data = response.get("version")
+ if not isinstance(version_data, dict):
+ raise RuntimeError(
+ f"crates.io returned invalid version metadata for {name} {version}"
+ )
+ return version_data
+
+ def list_github_configs(self, name: str) -> list[dict[str, object]]:
+ query = urllib.parse.urlencode({"crate": name, "per_page": 100})
+ response = self._request(
+ "GET",
+ f"/api/v1/trusted_publishing/github_configs?{query}",
+ f"listing Trusted Publishers for {name}",
+ authenticated=True,
+ )
+ configs = response.get("github_configs")
+ if not isinstance(configs, list) or not all(
+ isinstance(config, dict) for config in configs
+ ):
+ raise RuntimeError(
+ f"crates.io returned invalid Trusted Publisher metadata for {name}"
+ )
+ return configs
+
+ def create_github_config(self, name: str) -> dict[str, object]:
+ config = {"crate": name, **PUBLISHER}
+ response = self._request(
+ "POST",
+ "/api/v1/trusted_publishing/github_configs",
+ f"creating the Trusted Publisher for {name}",
+ body={"github_config": config},
+ authenticated=True,
+ )
+ created = response.get("github_config")
+ if not isinstance(created, dict):
+ raise RuntimeError(
+ f"crates.io returned invalid Trusted Publisher metadata for {name}"
+ )
+ return created
+
+ def set_trustpub_only(self, name: str) -> dict[str, object]:
+ encoded_name = urllib.parse.quote(name, safe="")
+ response = self._request(
+ "PATCH",
+ f"/api/v1/crates/{encoded_name}",
+ f"enabling Trusted Publishing only for {name}",
+ body={"crate": {"trustpub_only": True}},
+ authenticated=True,
+ )
+ krate = response.get("crate")
+ if not isinstance(krate, dict):
+ raise RuntimeError(f"crates.io returned invalid metadata for {name}")
+ return krate
+
+
+def _normalized_repository(value: object) -> str:
+ if not isinstance(value, str):
+ return ""
+ normalized = value.rstrip("/")
+ if normalized.endswith(".git"):
+ normalized = normalized[:-4]
+ return normalized.lower()
+
+
+def validate_crate_metadata(
+ planned: PlannedCrate, metadata: dict[str, object], client: CratesIoClient
+) -> None:
+ if metadata.get("id") != planned.name:
+ raise RuntimeError(
+ f"crate name mismatch for {planned.name}: got {metadata.get('id')!r}"
+ )
+ if _normalized_repository(metadata.get("repository")) != _normalized_repository(
+ REPOSITORY
+ ):
+ raise RuntimeError(
+ f"{planned.name} already exists with an unexpected repository: "
+ f"{metadata.get('repository')!r}"
+ )
+
+ if metadata.get("max_version") == PLACEHOLDER_VERSION:
+ if metadata.get("description") != PLACEHOLDER_DESCRIPTION:
+ raise RuntimeError(
+ f"{planned.name} has an unexpected {PLACEHOLDER_VERSION} placeholder"
+ )
+ version = client.get_version(planned.name, PLACEHOLDER_VERSION)
+ if version.get("num") != PLACEHOLDER_VERSION:
+ raise RuntimeError(
+ f"{planned.name} placeholder version could not be verified"
+ )
+
+
+def _is_expected_config(name: str, config: dict[str, object]) -> bool:
+ repository_owner = config.get("repository_owner")
+ repository_name = config.get("repository_name")
+ return all(
+ (
+ config.get("crate") == name,
+ isinstance(repository_owner, str)
+ and repository_owner.lower() == PUBLISHER["repository_owner"],
+ isinstance(repository_name, str)
+ and repository_name.lower() == PUBLISHER["repository_name"],
+ config.get("workflow_filename") == PUBLISHER["workflow_filename"],
+ config.get("environment") == PUBLISHER["environment"],
+ )
+ )
+
+
+def validate_github_configs(name: str, configs: list[dict[str, object]]) -> None:
+ if len(configs) != 1 or not _is_expected_config(name, configs[0]):
+ compact = [
+ {
+ key: config.get(key)
+ for key in (
+ "repository_owner",
+ "repository_name",
+ "workflow_filename",
+ "environment",
+ )
+ }
+ for config in configs
+ ]
+ raise RuntimeError(
+ f"{name} has unexpected Trusted Publisher configurations: "
+ f"{json.dumps(compact, sort_keys=True)}"
+ )
+
+
+def preflight_authenticated(
+ packages: list[PlannedCrate],
+ candidate_names: set[str],
+ client: CratesIoClient,
+) -> list[str]:
+ verified: list[str] = []
+ for planned in packages:
+ metadata = client.get_crate(planned.name)
+ is_candidate = planned.name in candidate_names
+ if metadata is None:
+ if not is_candidate:
+ raise RuntimeError(
+ f"{planned.name} is missing but was not selected for bootstrap"
+ )
+ verified.append(planned.name)
+ continue
+
+ validate_crate_metadata(planned, metadata, client)
+ is_placeholder = metadata.get("max_version") == PLACEHOLDER_VERSION
+ if is_placeholder != is_candidate:
+ state = "a placeholder" if is_placeholder else "an established crate"
+ raise RuntimeError(
+ f"{planned.name} is now {state}, which does not match discovery"
+ )
+
+ configs = client.list_github_configs(planned.name)
+ if is_placeholder:
+ if configs:
+ validate_github_configs(planned.name, configs)
+ else:
+ validate_github_configs(planned.name, configs)
+ if metadata.get("trustpub_only") is not True:
+ raise RuntimeError(
+ f"{planned.name} does not require Trusted Publishing"
+ )
+ verified.append(planned.name)
+ return verified
+
+
+def verify_authenticated(
+ packages: list[PlannedCrate], client: CratesIoClient
+) -> list[str]:
+ verified: list[str] = []
+ for planned in packages:
+ metadata = client.get_crate(planned.name)
+ if metadata is None:
+ raise RuntimeError(f"{planned.name} does not exist on crates.io")
+ validate_crate_metadata(planned, metadata, client)
+ validate_github_configs(planned.name, client.list_github_configs(planned.name))
+ if metadata.get("trustpub_only") is not True:
+ raise RuntimeError(f"{planned.name} does not require Trusted Publishing")
+ verified.append(planned.name)
+ return verified
+
+
+def _placeholder_manifest(name: str) -> str:
+ return f"""# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = {json.dumps(name)}
+version = {json.dumps(PLACEHOLDER_VERSION)}
+edition = "2024"
+rust-version = "1.85.0"
+description = {json.dumps(PLACEHOLDER_DESCRIPTION)}
+homepage = "https://docs.rs/reqsign"
+repository = {json.dumps(REPOSITORY)}
+license = "Apache-2.0"
+readme = "README.md"
+include = ["src/lib.rs", "README.md", "LICENSE", "NOTICE"]
+"""
+
+
+def _placeholder_readme(name: str) -> str:
+ return f"""# {name}
+
+This crate belongs to [Apache OpenDAL reqsign]({REPOSITORY}).
+
+Version {PLACEHOLDER_VERSION} reserves the crates.io package name for Apache
+OpenDAL reqsign. It is not an ASF software release, contains no implementation,
+and must not be used as a dependency.
+"""
+
+
+PLACEHOLDER_LIB = """// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![doc = include_str!("../README.md")]
+"""
+
+
+def write_placeholder_package(
+ project_dir: Path, planned: PlannedCrate, package_dir: Path
+) -> None:
+ source_dir = package_dir / "src"
+ source_dir.mkdir()
+ (package_dir / "Cargo.toml").write_text(
+ _placeholder_manifest(planned.name), encoding="utf-8"
+ )
+ (package_dir / "README.md").write_text(
+ _placeholder_readme(planned.name), encoding="utf-8"
+ )
+ (source_dir / "lib.rs").write_text(PLACEHOLDER_LIB, encoding="utf-8")
+ shutil.copyfile(project_dir / "LICENSE", package_dir / "LICENSE")
+ shutil.copyfile(project_dir / "NOTICE", package_dir / "NOTICE")
+
+
+def publish_placeholder(project_dir: Path, planned: PlannedCrate, token: str) -> None:
+ with tempfile.TemporaryDirectory(prefix=f"{planned.name}-bootstrap-") as tmpdir:
+ package_dir = Path(tmpdir)
+ write_placeholder_package(project_dir, planned, package_dir)
+
+ env = os.environ.copy()
+ env["CARGO_REGISTRY_TOKEN"] = token
+ command = ["cargo", "publish", "--manifest-path", "Cargo.toml"]
+ while True:
+ process = subprocess.run(
+ command,
+ cwd=package_dir,
+ env=env,
+ check=False,
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ )
+ output = process.stdout or ""
+ print(output, end="", flush=True)
+ if process.returncode == 0:
+ return
+ if should_retry(output):
+ delay = parse_retry_after(output)
+ print(
+ f"crates.io rate limited {planned.name}; sleeping {delay}s",
+ flush=True,
+ )
+ time.sleep(delay)
+ continue
+ raise subprocess.CalledProcessError(
+ process.returncode, command, output=output
+ )
+
+
+def wait_for_crate(
+ client: CratesIoClient, name: str, timeout: int = 120
+) -> dict[str, object]:
+ deadline = time.monotonic() + timeout
+ while True:
+ metadata = client.get_crate(name)
+ if metadata is not None:
+ return metadata
+ if time.monotonic() >= deadline:
+ raise RuntimeError(
+ f"timed out waiting for {name} to become visible on crates.io"
+ )
+ time.sleep(2)
+
+
+def wait_for_trustpub_only(
+ client: CratesIoClient, name: str, timeout: int = 120
+) -> dict[str, object]:
+ deadline = time.monotonic() + timeout
+ while True:
+ metadata = client.get_crate(name)
+ if metadata is not None and metadata.get("trustpub_only") is True:
+ return metadata
+ if time.monotonic() >= deadline:
+ raise RuntimeError(
+ f"timed out waiting for {name} to require Trusted Publishing"
+ )
+ time.sleep(2)
+
+
+def wait_for_expected_config(
+ client: CratesIoClient, name: str, timeout: int = 120
+) -> None:
+ deadline = time.monotonic() + timeout
+ while True:
+ configs = client.list_github_configs(name)
+ if configs:
+ validate_github_configs(name, configs)
+ return
+ if time.monotonic() >= deadline:
+ raise RuntimeError(
+ f"timed out waiting for the Trusted Publisher for {name}"
+ )
+ time.sleep(2)
+
+
+def reconcile_crate(
+ project_dir: Path,
+ planned: PlannedCrate,
+ client: CratesIoClient,
+ token: str,
+) -> ReconcileResult:
+ actions: list[str] = []
+ metadata = client.get_crate(planned.name)
+ if metadata is None:
+ publish_placeholder(project_dir, planned, token)
+ metadata = wait_for_crate(client, planned.name)
+ actions.append("created placeholder")
+ elif metadata.get("max_version") != PLACEHOLDER_VERSION:
+ raise RuntimeError(
+ f"{planned.name} became an established crate after discovery; "
+ "refusing to modify it in the bootstrap workflow"
+ )
+
+ validate_crate_metadata(planned, metadata, client)
+
+ configs = client.list_github_configs(planned.name)
+ if not configs:
+ created = client.create_github_config(planned.name)
+ if not _is_expected_config(planned.name, created):
+ raise RuntimeError(
+ f"crates.io created an unexpected Trusted Publisher for {planned.name}"
+ )
+ actions.append("configured Trusted Publishing")
+ else:
+ validate_github_configs(planned.name, configs)
+
+ if metadata.get("trustpub_only") is not True:
+ updated = client.set_trustpub_only(planned.name)
+ if updated.get("trustpub_only") is not True:
+ raise RuntimeError(
+ f"crates.io did not enable Trusted Publishing only for {planned.name}"
+ )
+ actions.append("enabled Trusted Publishing only")
+
+ verified_metadata = wait_for_trustpub_only(client, planned.name)
+ validate_crate_metadata(planned, verified_metadata, client)
+ wait_for_expected_config(client, planned.name)
+
+ if not actions:
+ actions.append("verified")
+ return ReconcileResult(planned.name, tuple(actions))
+
+
+def discover(
+ project_dir: Path, client: CratesIoClient
+) -> tuple[list[PlannedCrate], list[str], list[str]]:
+ packages = planned_crates(project_dir)
+ missing: list[str] = []
+ placeholders: list[str] = []
+ for planned in packages:
+ metadata = client.get_crate(planned.name)
+ if metadata is None:
+ missing.append(planned.name)
+ continue
+ validate_crate_metadata(planned, metadata, client)
+ if metadata.get("max_version") == PLACEHOLDER_VERSION:
+ placeholders.append(planned.name)
+ return packages, missing, placeholders
+
+
+def verify_public(project_dir: Path, client: CratesIoClient) -> list[str]:
+ verified: list[str] = []
+ for planned in planned_crates(project_dir):
+ metadata = client.get_crate(planned.name)
+ if metadata is None:
+ raise RuntimeError(f"{planned.name} does not exist on crates.io")
+ validate_crate_metadata(planned, metadata, client)
+ if metadata.get("trustpub_only") is not True:
+ raise RuntimeError(f"{planned.name} does not require Trusted Publishing")
+ verified.append(planned.name)
+ return verified
+
+
+def _write_summary(title: str, lines: list[str]) -> None:
+ summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+ if not summary_path:
+ return
+ source_commit = os.environ.get("GITHUB_SHA", "unknown")
+ with Path(summary_path).open("a", encoding="utf-8") as summary:
+ summary.write(f"## {title}\n\n")
+ summary.write(f"Source commit: `{source_commit}`\n\n")
+ for line in lines:
+ summary.write(f"- {line}\n")
+ summary.write("\n")
+
+
+def run_discover(args: argparse.Namespace) -> int:
+ client = CratesIoClient(args.registry_url)
+ packages, missing, placeholders = discover(args.project_dir, client)
+ candidates = [*missing, *placeholders]
+ result = {
+ "packages": len(packages),
+ "missing": missing,
+ "placeholders": placeholders,
+ "bootstrap_candidates": candidates,
+ }
+ print(json.dumps(result, indent=2))
+ _write_summary(
+ "Rust crate bootstrap discovery",
+ [
+ f"Publishable crates: {len(packages)}",
+ f"Missing crates: {', '.join(missing) if missing else 'none'}",
+ f"Placeholder crates: {', '.join(placeholders) if placeholders else 'none'}",
+ ],
+ )
+ return 0
+
+
+def run_apply(args: argparse.Namespace) -> int:
+ token = os.environ.get("CARGO_REGISTRY_BOOTSTRAP_TOKEN")
+ if not token:
+ raise RuntimeError("CARGO_REGISTRY_BOOTSTRAP_TOKEN is not set")
+
+ client = CratesIoClient(args.registry_url, token=token)
+ packages, missing, placeholders = discover(args.project_dir, client)
+ candidate_set = {*missing, *placeholders}
+ candidates = [package for package in packages if package.name in candidate_set]
+
+ results: list[ReconcileResult] = []
+ authenticated: list[str] = []
+ try:
+ preflight_authenticated(packages, candidate_set, client)
+ print(
+ f"authenticated preflight passed for {len(packages)} planned crates",
+ flush=True,
+ )
+ print(f"bootstrap candidates: {len(candidates)}", flush=True)
+ for planned in candidates:
+ result = reconcile_crate(args.project_dir, planned, client, token)
+ results.append(result)
+ print(f"{result.name}: {', '.join(result.actions)}", flush=True)
+ authenticated = verify_authenticated(packages, client)
+ print(
+ f"authenticated final audit passed for {len(authenticated)} planned crates",
+ flush=True,
+ )
+ except Exception as error:
+ _write_summary(
+ "Rust crate bootstrap",
+ [
+ *(
+ f"`{result.name}`: {', '.join(result.actions)}"
+ for result in results
+ ),
+ f"Failed: {error}",
+ ],
+ )
+ raise
+
+ _write_summary(
+ "Rust crate bootstrap",
+ [
+ f"Authenticated preflight: {len(packages)} planned crates",
+ *(f"`{result.name}`: {', '.join(result.actions)}" for result in results),
+ f"Authenticated final audit: {len(authenticated)} planned crates",
+ ],
+ )
+ return 0
+
+
+def run_verify(args: argparse.Namespace) -> int:
+ verified = verify_public(args.project_dir, CratesIoClient(args.registry_url))
+ print(json.dumps({"verified": verified}, indent=2))
+ return 0
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ description=(
+ "Create and secure crates.io names in the reqsign Rust publish plan."
+ )
+ )
+ parser.add_argument(
+ "--project-dir",
+ type=Path,
+ default=PROJECT_DIR,
+ help="Path to the repository root.",
+ )
+ parser.add_argument(
+ "--registry-url",
+ default=REGISTRY_URL,
+ help="crates.io-compatible registry API URL.",
+ )
+ subparsers = parser.add_subparsers(dest="command", required=True)
+
+ discover_parser = subparsers.add_parser(
+ "discover", help="Report missing and placeholder crate names."
+ )
+ discover_parser.set_defaults(run=run_discover)
+
+ apply_parser = subparsers.add_parser(
+ "apply",
+ help="Audit all planned crates and reconcile missing names and placeholders.",
+ )
+ apply_parser.set_defaults(run=run_apply)
+
+ verify_parser = subparsers.add_parser(
+ "verify", help="Verify public crate existence and Trusted Publishing only."
+ )
+ verify_parser.set_defaults(run=run_verify)
+
+ args = parser.parse_args()
+ args.project_dir = args.project_dir.resolve()
+ return args.run(args)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/scripts/release_rust/plan.py b/.github/scripts/release_rust/plan.py
new file mode 100644
index 0000000..5654a6e
--- /dev/null
+++ b/.github/scripts/release_rust/plan.py
@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import heapq
+import json
+import subprocess
+from dataclasses import asdict
+from dataclasses import dataclass
+from pathlib import Path
+
+
+SCRIPT_PATH = Path(__file__).resolve()
+PROJECT_DIR = SCRIPT_PATH.parents[3]
+
+
+@dataclass(frozen=True)
+class Package:
+ name: str
+ version: str
+ path: str
+
+
+def load_metadata(project_dir: Path = PROJECT_DIR) -> dict[str, object]:
+ process = subprocess.run(
+ ["cargo", "metadata", "--no-deps", "--format-version", "1"],
+ cwd=project_dir,
+ check=True,
+ text=True,
+ stdout=subprocess.PIPE,
+ )
+ metadata = json.loads(process.stdout)
+ if not isinstance(metadata, dict):
+ raise RuntimeError("cargo metadata returned an invalid document")
+ return metadata
+
+
+def is_publishable(package: dict[str, object]) -> bool:
+ publish = package.get("publish")
+ if publish is None:
+ return True
+ if not isinstance(publish, list):
+ raise RuntimeError(
+ f"cargo metadata returned invalid publish settings for {package.get('name')}"
+ )
+ return "crates-io" in publish
+
+
+def plan_from_metadata(metadata: dict[str, object], project_dir: Path) -> list[Package]:
+ raw_packages = metadata.get("packages")
+ workspace_members = metadata.get("workspace_members")
+ if not isinstance(raw_packages, list) or not isinstance(workspace_members, list):
+ raise RuntimeError("cargo metadata omitted packages or workspace members")
+
+ members = set(workspace_members)
+ packages_by_id: dict[str, dict[str, object]] = {}
+ publishable_by_dir: dict[Path, dict[str, object]] = {}
+ local_by_dir: dict[Path, dict[str, object]] = {}
+
+ for package in raw_packages:
+ if not isinstance(package, dict):
+ raise RuntimeError("cargo metadata returned an invalid package")
+ package_id = package.get("id")
+ manifest_path = package.get("manifest_path")
+ if not isinstance(package_id, str) or not isinstance(manifest_path, str):
+ raise RuntimeError("cargo metadata returned an invalid package identity")
+ if package_id not in members:
+ continue
+
+ manifest_dir = Path(manifest_path).resolve().parent
+ packages_by_id[package_id] = package
+ local_by_dir[manifest_dir] = package
+ if is_publishable(package):
+ publishable_by_dir[manifest_dir] = package
+
+ if set(packages_by_id) != members:
+ missing = sorted(members - set(packages_by_id))
+ raise RuntimeError(f"cargo metadata omitted workspace packages: {missing}")
+
+ graph: dict[Path, set[Path]] = {
+ manifest_dir: set() for manifest_dir in publishable_by_dir
+ }
+ indegree = {manifest_dir: 0 for manifest_dir in publishable_by_dir}
+
+ for manifest_dir, package in publishable_by_dir.items():
+ dependencies = package.get("dependencies")
+ if not isinstance(dependencies, list):
+ raise RuntimeError(
+ f"cargo metadata omitted dependencies for {package.get('name')}"
+ )
+
+ for dependency in dependencies:
+ if not isinstance(dependency, dict):
+ raise RuntimeError("cargo metadata returned an invalid dependency")
+ if dependency.get("kind") == "dev":
+ continue
+ dependency_path = dependency.get("path")
+ if not isinstance(dependency_path, str):
+ continue
+
+ dependency_dir = Path(dependency_path).resolve()
+ if dependency_dir not in local_by_dir:
+ continue
+ if dependency_dir not in publishable_by_dir:
+ raise RuntimeError(
+ f"{package.get('name')} depends on unpublished workspace package "
+ f"{local_by_dir[dependency_dir].get('name')}"
+ )
+ if manifest_dir in graph[dependency_dir]:
+ continue
+
+ graph[dependency_dir].add(manifest_dir)
+ indegree[manifest_dir] += 1
+
+ queue = [
+ (manifest_dir.relative_to(project_dir).as_posix(), manifest_dir)
+ for manifest_dir, degree in indegree.items()
+ if degree == 0
+ ]
+ heapq.heapify(queue)
+
+ ordered: list[Package] = []
+ while queue:
+ _, manifest_dir = heapq.heappop(queue)
+ package = publishable_by_dir[manifest_dir]
+ name = package.get("name")
+ version = package.get("version")
+ if not isinstance(name, str) or not isinstance(version, str):
+ raise RuntimeError("cargo metadata returned invalid package metadata")
+
+ ordered.append(
+ Package(
+ name=name,
+ version=version,
+ path=manifest_dir.relative_to(project_dir).as_posix(),
+ )
+ )
+ for dependent in graph[manifest_dir]:
+ indegree[dependent] -= 1
+ if indegree[dependent] == 0:
+ heapq.heappush(
+ queue,
+ (dependent.relative_to(project_dir).as_posix(), dependent),
+ )
+
+ if len(ordered) != len(publishable_by_dir):
+ cyclic = sorted(
+ publishable_by_dir[path].get("name")
+ for path, degree in indegree.items()
+ if degree > 0
+ )
+ raise RuntimeError(f"publishable workspace dependency cycle: {cyclic}")
+
+ names = [package.name for package in ordered]
+ if len(names) != len(set(names)):
+ raise RuntimeError("duplicate crates.io package name in publish plan")
+ return ordered
+
+
+def plan(project_dir: Path = PROJECT_DIR) -> list[Package]:
+ project_dir = project_dir.resolve()
+ return plan_from_metadata(load_metadata(project_dir), project_dir)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ description="Print the reqsign crates.io publish plan in dependency order."
+ )
+ parser.add_argument(
+ "--project-dir",
+ type=Path,
+ default=PROJECT_DIR,
+ help="Path to the repository root.",
+ )
+ args = parser.parse_args()
+ print(json.dumps([asdict(package) for package in plan(args.project_dir)], indent=2))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/scripts/release_rust/publish.py b/.github/scripts/release_rust/publish.py
new file mode 100644
index 0000000..00671ca
--- /dev/null
+++ b/.github/scripts/release_rust/publish.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import os
+import re
+import subprocess
+import time
+from datetime import datetime
+from datetime import timezone
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+
+from plan import PROJECT_DIR
+from plan import Package
+from plan import plan
+from trusted_publishing import temporary_trusted_publishing_token
+
+
+def parse_retry_after(output: str) -> int:
+ match = re.search(r"Please try again after ([^\n]+)", output)
+ if match:
+ value = match.group(1).strip().rstrip(".")
+ for parser in (
+ lambda text: parsedate_to_datetime(text),
+ lambda text: datetime.fromisoformat(text.replace("Z", "+00:00")),
+ ):
+ try:
+ retry_at = parser(value)
+ if retry_at.tzinfo is None:
+ retry_at = retry_at.replace(tzinfo=timezone.utc)
+ return max(
+ 60,
+ int((retry_at - datetime.now(timezone.utc)).total_seconds()) + 8,
+ )
+ except ValueError:
+ continue
+ return 610
+
+
+def should_retry(output: str) -> bool:
+ lowered = output.lower()
+ return (
+ "too many requests" in lowered
+ or "rate limit" in lowered
+ or "you have published too many crates" in lowered
+ )
+
+
+def already_published(output: str, package: Package) -> bool:
+ lowered = output.lower()
+ identity = f"{package.name}@{package.version}".lower()
+ return identity in lowered and (
+ "already exists" in lowered or "already uploaded" in lowered
+ )
+
+
+def publish_package(project_dir: Path, package: Package) -> str:
+ command = [
+ "cargo",
+ "publish",
+ "--package",
+ package.name,
+ "--no-verify",
+ ]
+
+ while True:
+ print(f"Publishing {package.name} {package.version}", flush=True)
+ with temporary_trusted_publishing_token() as token:
+ env = os.environ.copy()
+ env["CARGO_REGISTRY_TOKEN"] = token
+ process = subprocess.run(
+ command,
+ cwd=project_dir,
+ check=False,
+ env=env,
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ )
+ output = process.stdout or ""
+ print(output, end="", flush=True)
+
+ if process.returncode == 0:
+ return "published"
+ if already_published(output, package):
+ print(
+ f"Skipping {package.name} {package.version}: already published",
+ flush=True,
+ )
+ return "already published"
+ if should_retry(output):
+ delay = parse_retry_after(output)
+ print(
+ f"crates.io rate limited {package.name}; sleeping {delay}s",
+ flush=True,
+ )
+ time.sleep(delay)
+ continue
+ raise subprocess.CalledProcessError(process.returncode, command, output=output)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ description="Publish reqsign crates with fresh OIDC credentials."
+ )
+ parser.add_argument(
+ "--project-dir",
+ type=Path,
+ default=PROJECT_DIR,
+ help="Path to the repository root.",
+ )
+ args = parser.parse_args()
+ project_dir = args.project_dir.resolve()
+
+ results: list[tuple[Package, str]] = []
+ for package in plan(project_dir):
+ results.append((package, publish_package(project_dir, package)))
+
+ print("Publish plan completed:", flush=True)
+ for package, result in results:
+ print(f"- {package.name} {package.version}: {result}", flush=True)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/scripts/release_rust/test_bootstrap.py b/.github/scripts/release_rust/test_bootstrap.py
new file mode 100644
index 0000000..1533453
--- /dev/null
+++ b/.github/scripts/release_rust/test_bootstrap.py
@@ -0,0 +1,456 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import io
+import subprocess
+import tempfile
+import tomllib
+import unittest
+import urllib.error
+from pathlib import Path
+from unittest import mock
+
+from bootstrap import CratesIoClient
+from bootstrap import PLACEHOLDER_DESCRIPTION
+from bootstrap import PLACEHOLDER_VERSION
+from bootstrap import PUBLISHER
+from bootstrap import PROJECT_DIR
+from bootstrap import REPOSITORY
+from bootstrap import PlannedCrate
+from bootstrap import ReconcileResult
+from bootstrap import _placeholder_manifest
+from bootstrap import discover
+from bootstrap import preflight_authenticated
+from bootstrap import reconcile_crate
+from bootstrap import run_apply
+from bootstrap import verify_authenticated
+from bootstrap import write_placeholder_package
+
+
+def metadata(name: str, *, version: str = "1.0.0", trustpub_only: bool = False):
+ return {
+ "id": name,
+ "max_version": version,
+ "repository": REPOSITORY,
+ "description": (
+ PLACEHOLDER_DESCRIPTION
+ if version == PLACEHOLDER_VERSION
+ else "An Apache OpenDAL reqsign crate"
+ ),
+ "trustpub_only": trustpub_only,
+ }
+
+
+def expected_config(name: str):
+ return {"crate": name, **PUBLISHER}
+
+
+class FakeClient:
+ def __init__(self, name: str, krate=None, configs=None):
+ self.name = name
+ self.krate = krate
+ self.configs = list(configs or [])
+ self.created_configs = 0
+ self.restricted = 0
+
+ def get_crate(self, name: str):
+ self._assert_name(name)
+ return None if self.krate is None else dict(self.krate)
+
+ def get_version(self, name: str, version: str):
+ self._assert_name(name)
+ return {"num": version}
+
+ def list_github_configs(self, name: str):
+ self._assert_name(name)
+ return [dict(config) for config in self.configs]
+
+ def create_github_config(self, name: str):
+ self._assert_name(name)
+ config = expected_config(name)
+ self.configs.append(config)
+ self.created_configs += 1
+ return dict(config)
+
+ def set_trustpub_only(self, name: str):
+ self._assert_name(name)
+ self.krate["trustpub_only"] = True
+ self.restricted += 1
+ return dict(self.krate)
+
+ def _assert_name(self, name: str):
+ if name != self.name:
+ raise AssertionError(f"expected {self.name}, got {name}")
+
+
+class JsonResponse(io.BytesIO):
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+
+class BootstrapTest(unittest.TestCase):
+ def test_bootstrap_workflow_is_input_free_and_always_protected(self):
+ workflow = (
+ PROJECT_DIR / ".github/workflows/bootstrap_rust_crates.yml"
+ ).read_text(encoding="utf-8")
+ dispatch = workflow.split("\non:\n", 1)[1].split("\npermissions:\n", 1)[0]
+
+ self.assertEqual(dispatch, " workflow_dispatch:\n")
+ self.assertIn(" environment: rust-bootstrap\n", workflow)
+ self.assertNotIn("candidate_count", workflow)
+
+ def test_publisher_matches_the_release_workflow(self):
+ workflow = (PROJECT_DIR / ".github/workflows/release.yml").read_text(
+ encoding="utf-8"
+ )
+ publish_job = workflow.split("\n publish:\n", 1)[1]
+
+ self.assertEqual(PUBLISHER["workflow_filename"], "release.yml")
+ self.assertEqual(PUBLISHER["environment"], "release")
+ self.assertIn(" environment: release\n", publish_job)
+ self.assertIn("github.event_name == 'push'", publish_job)
+ self.assertIn("!contains(github.ref, '-')", publish_job)
+ self.assertIn(" id-token: write\n", publish_job)
+ self.assertNotIn("secrets.CARGO_REGISTRY_TOKEN", publish_job)
+ self.assertNotIn("CARGO_REGISTRY_BOOTSTRAP_TOKEN", workflow)
+
+ def test_placeholder_manifest_is_dependency_free(self):
+ manifest = tomllib.loads(_placeholder_manifest("reqsign-new"))
+
+ self.assertEqual(manifest["package"]["name"], "reqsign-new")
+ self.assertEqual(manifest["package"]["version"], PLACEHOLDER_VERSION)
+ self.assertNotIn("dependencies", manifest)
+ self.assertNotIn("dev-dependencies", manifest)
+ self.assertNotIn("build-dependencies", manifest)
+
+ def test_placeholder_can_be_packaged_offline(self):
+ planned = PlannedCrate("reqsign-new", "services/new")
+ with tempfile.TemporaryDirectory() as tmpdir:
+ package_dir = Path(tmpdir)
+ write_placeholder_package(PROJECT_DIR, planned, package_dir)
+
+ process = subprocess.run(
+ [
+ "cargo",
+ "package",
+ "--manifest-path",
+ str(package_dir / "Cargo.toml"),
+ "--no-verify",
+ "--offline",
+ ],
+ check=False,
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ )
+
+ self.assertEqual(process.returncode, 0, process.stdout)
+ self.assertTrue(
+ (
+ package_dir
+ / "target"
+ / "package"
+ / f"{planned.name}-{PLACEHOLDER_VERSION}.crate"
+ ).is_file()
+ )
+
+ def test_public_read_retries_a_transient_registry_failure(self):
+ error = urllib.error.HTTPError(
+ "https://crates.io/api/v1/crates/reqsign",
+ 503,
+ "unavailable",
+ {},
+ io.BytesIO(b""),
+ )
+ response = JsonResponse(
+ b'{"crate":{"id":"reqsign","repository":'
+ b'"https://github.com/apache/opendal-reqsign"}}'
+ )
+
+ with (
+ mock.patch(
+ "bootstrap.urllib.request.urlopen",
+ side_effect=(error, response),
+ ),
+ mock.patch("bootstrap.time.sleep") as sleep,
+ ):
+ krate = CratesIoClient().get_crate("reqsign")
+
+ self.assertEqual(krate["id"], "reqsign")
+ sleep.assert_called_once_with(1)
+
+ def test_discovery_does_not_migrate_established_crates(self):
+ planned = PlannedCrate("reqsign-core", "core")
+ client = FakeClient(planned.name, metadata(planned.name))
+
+ with mock.patch("bootstrap.planned_crates", return_value=[planned]):
+ packages, missing, placeholders = discover(Path(), client)
+
+ self.assertEqual(packages, [planned])
+ self.assertEqual(missing, [])
+ self.assertEqual(placeholders, [])
+
+ def test_discovery_selects_a_placeholder(self):
+ planned = PlannedCrate("reqsign-new", "services/new")
+ client = FakeClient(
+ planned.name,
+ metadata(planned.name, version=PLACEHOLDER_VERSION),
+ )
+
+ with mock.patch("bootstrap.planned_crates", return_value=[planned]):
+ _, missing, placeholders = discover(Path(), client)
+
+ self.assertEqual(missing, [])
+ self.assertEqual(placeholders, [planned.name])
+
+ def test_authenticated_preflight_audits_an_established_crate(self):
+ planned = PlannedCrate("reqsign-core", "core")
+ client = FakeClient(
+ planned.name,
+ metadata(planned.name, trustpub_only=True),
+ [expected_config(planned.name)],
+ )
+
+ verified = preflight_authenticated([planned], set(), client)
+
+ self.assertEqual(verified, [planned.name])
+ self.assertEqual(client.created_configs, 0)
+ self.assertEqual(client.restricted, 0)
+
+ def test_authenticated_preflight_rejects_an_unexpected_publisher(self):
+ planned = PlannedCrate("reqsign-core", "core")
+ unexpected = {
+ **expected_config(planned.name),
+ "workflow_filename": "other.yml",
+ }
+ client = FakeClient(
+ planned.name,
+ metadata(planned.name, trustpub_only=True),
+ [unexpected],
+ )
+
+ with self.assertRaisesRegex(RuntimeError, "unexpected Trusted Publisher"):
+ preflight_authenticated([planned], set(), client)
+
+ self.assertEqual(client.created_configs, 0)
+ self.assertEqual(client.restricted, 0)
+
+ def test_authenticated_preflight_rejects_unmigrated_established_crate(self):
+ planned = PlannedCrate("reqsign-core", "core")
+ client = FakeClient(
+ planned.name,
+ metadata(planned.name),
+ [expected_config(planned.name)],
+ )
+
+ with self.assertRaisesRegex(RuntimeError, "does not require"):
+ preflight_authenticated([planned], set(), client)
+
+ def test_authenticated_verification_requires_the_exact_publisher(self):
+ planned = PlannedCrate("reqsign-core", "core")
+ unexpected = {
+ **expected_config(planned.name),
+ "environment": "other",
+ }
+ client = FakeClient(
+ planned.name,
+ metadata(planned.name, trustpub_only=True),
+ [unexpected],
+ )
+
+ with self.assertRaisesRegex(RuntimeError, "unexpected Trusted Publisher"):
+ verify_authenticated([planned], client)
+
+ def test_apply_preflights_the_complete_plan_before_reconciling(self):
+ candidate = PlannedCrate("reqsign-new", "services/new")
+ established = PlannedCrate("reqsign-core", "core")
+ packages = [candidate, established]
+ client = object()
+ calls = []
+
+ def preflight(actual_packages, candidate_names, actual_client):
+ calls.append(("preflight", actual_packages, candidate_names, actual_client))
+
+ def reconcile(project_dir, planned, actual_client, token):
+ calls.append(("reconcile", planned, actual_client, token))
+ return ReconcileResult(planned.name, ("verified",))
+
+ def verify(actual_packages, actual_client):
+ calls.append(("verify", actual_packages, actual_client))
+ return [package.name for package in actual_packages]
+
+ args = argparse.Namespace(project_dir=Path(), registry_url="registry")
+ with (
+ mock.patch.dict(
+ "bootstrap.os.environ",
+ {"CARGO_REGISTRY_BOOTSTRAP_TOKEN": "bootstrap-token"},
+ clear=True,
+ ),
+ mock.patch("bootstrap.CratesIoClient", return_value=client),
+ mock.patch(
+ "bootstrap.discover",
+ return_value=(packages, [candidate.name], []),
+ ),
+ mock.patch("bootstrap.preflight_authenticated", preflight),
+ mock.patch("bootstrap.reconcile_crate", reconcile),
+ mock.patch("bootstrap.verify_authenticated", verify),
+ ):
+ self.assertEqual(run_apply(args), 0)
+
+ self.assertEqual(
+ [call[0] for call in calls],
+ ["preflight", "reconcile", "verify"],
+ )
+ self.assertEqual(calls[0][1], packages)
+ self.assertEqual(calls[0][2], {candidate.name})
+ self.assertEqual(calls[2][1], packages)
+
+ def test_apply_does_not_mutate_when_authenticated_preflight_fails(self):
+ candidate = PlannedCrate("reqsign-new", "services/new")
+ args = argparse.Namespace(project_dir=Path(), registry_url="registry")
+
+ with (
+ mock.patch.dict(
+ "bootstrap.os.environ",
+ {"CARGO_REGISTRY_BOOTSTRAP_TOKEN": "bootstrap-token"},
+ clear=True,
+ ),
+ mock.patch("bootstrap.CratesIoClient"),
+ mock.patch(
+ "bootstrap.discover",
+ return_value=([candidate], [candidate.name], []),
+ ),
+ mock.patch(
+ "bootstrap.preflight_authenticated",
+ side_effect=RuntimeError("audit failed"),
+ ),
+ mock.patch("bootstrap.reconcile_crate") as reconcile,
+ mock.patch("bootstrap.verify_authenticated") as verify,
+ self.assertRaisesRegex(RuntimeError, "audit failed"),
+ ):
+ run_apply(args)
+
+ reconcile.assert_not_called()
+ verify.assert_not_called()
+
+ def test_missing_crate_is_created_configured_and_restricted(self):
+ planned = PlannedCrate("reqsign-new", "services/new")
+ client = FakeClient(planned.name)
+
+ def publish(project_dir, package, token):
+ self.assertEqual(package, planned)
+ self.assertEqual(token, "bootstrap-token")
+ client.krate = metadata(planned.name, version=PLACEHOLDER_VERSION)
+
+ with (
+ tempfile.TemporaryDirectory() as tmpdir,
+ mock.patch("bootstrap.publish_placeholder", publish),
+ ):
+ result = reconcile_crate(Path(tmpdir), planned, client, "bootstrap-token")
+
+ self.assertEqual(
+ result.actions,
+ (
+ "created placeholder",
+ "configured Trusted Publishing",
+ "enabled Trusted Publishing only",
+ ),
+ )
+ self.assertEqual(client.created_configs, 1)
+ self.assertEqual(client.restricted, 1)
+
+ def test_partial_placeholder_resumes_without_republishing(self):
+ planned = PlannedCrate("reqsign-new", "services/new")
+ client = FakeClient(
+ planned.name,
+ metadata(planned.name, version=PLACEHOLDER_VERSION),
+ )
+
+ with (
+ tempfile.TemporaryDirectory() as tmpdir,
+ mock.patch("bootstrap.publish_placeholder") as publish,
+ ):
+ result = reconcile_crate(Path(tmpdir), planned, client, "bootstrap-token")
+
+ publish.assert_not_called()
+ self.assertEqual(
+ result.actions,
+ (
+ "configured Trusted Publishing",
+ "enabled Trusted Publishing only",
+ ),
+ )
+
+ def test_ready_placeholder_is_a_noop(self):
+ planned = PlannedCrate("reqsign-new", "services/new")
+ client = FakeClient(
+ planned.name,
+ metadata(
+ planned.name,
+ version=PLACEHOLDER_VERSION,
+ trustpub_only=True,
+ ),
+ [expected_config(planned.name)],
+ )
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ result = reconcile_crate(Path(tmpdir), planned, client, "bootstrap-token")
+
+ self.assertEqual(result.actions, ("verified",))
+ self.assertEqual(client.created_configs, 0)
+ self.assertEqual(client.restricted, 0)
+
+ def test_established_crate_is_never_migrated(self):
+ planned = PlannedCrate("reqsign-core", "core")
+ client = FakeClient(planned.name, metadata(planned.name))
+
+ with (
+ tempfile.TemporaryDirectory() as tmpdir,
+ self.assertRaisesRegex(RuntimeError, "established crate"),
+ ):
+ reconcile_crate(Path(tmpdir), planned, client, "bootstrap-token")
+
+ self.assertEqual(client.created_configs, 0)
+ self.assertEqual(client.restricted, 0)
+
+ def test_unexpected_publisher_fails_closed(self):
+ planned = PlannedCrate("reqsign-new", "services/new")
+ unexpected = {
+ **expected_config(planned.name),
+ "workflow_filename": "other.yml",
+ }
+ client = FakeClient(
+ planned.name,
+ metadata(planned.name, version=PLACEHOLDER_VERSION),
+ [unexpected],
+ )
+
+ with (
+ tempfile.TemporaryDirectory() as tmpdir,
+ self.assertRaisesRegex(RuntimeError, "unexpected Trusted Publisher"),
+ ):
+ reconcile_crate(Path(tmpdir), planned, client, "bootstrap-token")
+
+ self.assertEqual(client.restricted, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/release_rust/test_plan.py b/.github/scripts/release_rust/test_plan.py
new file mode 100644
index 0000000..c064094
--- /dev/null
+++ b/.github/scripts/release_rust/test_plan.py
@@ -0,0 +1,144 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import tempfile
+import unittest
+from pathlib import Path
+
+from plan import PROJECT_DIR
+from plan import Package
+from plan import plan
+from plan import plan_from_metadata
+
+
+def package(
+ root: Path,
+ name: str,
+ *,
+ dependencies: list[dict[str, object]] | None = None,
+ publish: list[str] | None = None,
+) -> dict[str, object]:
+ return {
+ "id": f"path+file://{root / name}#{name}@1.0.0",
+ "name": name,
+ "version": "1.0.0",
+ "manifest_path": str(root / name / "Cargo.toml"),
+ "publish": publish,
+ "dependencies": dependencies or [],
+ }
+
+
+def dependency(root: Path, name: str, *, kind: str | None = None):
+ return {
+ "name": name,
+ "kind": kind,
+ "path": str(root / name),
+ }
+
+
+class ReleaseRustPlanTest(unittest.TestCase):
+ def test_current_workspace_plan_contains_new_aws_crates(self):
+ packages = plan(PROJECT_DIR)
+ positions = {package.name: index for index, package in enumerate(packages)}
+
+ self.assertIn("reqsign-aws-core", positions)
+ self.assertIn("reqsign-aws-v4a", positions)
+ self.assertLess(positions["reqsign-core"], positions["reqsign-aws-core"])
+ self.assertLess(positions["reqsign-aws-core"], positions["reqsign-aws-v4"])
+ self.assertLess(positions["reqsign-aws-core"], positions["reqsign-aws-v4a"])
+ self.assertLess(positions["reqsign-aws-v4"], positions["reqsign-google"])
+ self.assertEqual(packages[-1].name, "reqsign")
+
+ def test_plan_is_topological_and_ignores_dev_dependencies(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ root = Path(tmpdir).resolve()
+ core = package(root, "core")
+ service = package(
+ root,
+ "service",
+ dependencies=[dependency(root, "core")],
+ )
+ facade = package(
+ root,
+ "facade",
+ dependencies=[
+ dependency(root, "service"),
+ dependency(root, "dev-only", kind="dev"),
+ ],
+ )
+ dev_only = package(root, "dev-only")
+ packages = [facade, service, dev_only, core]
+ metadata = {
+ "packages": packages,
+ "workspace_members": [package["id"] for package in packages],
+ }
+
+ result = plan_from_metadata(metadata, root)
+
+ self.assertEqual(
+ result,
+ [
+ Package("core", "1.0.0", "core"),
+ Package("dev-only", "1.0.0", "dev-only"),
+ Package("service", "1.0.0", "service"),
+ Package("facade", "1.0.0", "facade"),
+ ],
+ )
+
+ def test_unpublishable_workspace_dependency_fails_closed(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ root = Path(tmpdir).resolve()
+ internal = package(root, "internal", publish=[])
+ public = package(
+ root,
+ "public",
+ dependencies=[dependency(root, "internal")],
+ )
+ packages = [internal, public]
+ metadata = {
+ "packages": packages,
+ "workspace_members": [package["id"] for package in packages],
+ }
+
+ with self.assertRaisesRegex(RuntimeError, "unpublished workspace package"):
+ plan_from_metadata(metadata, root)
+
+ def test_cycle_fails_closed(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ root = Path(tmpdir).resolve()
+ first = package(
+ root,
+ "first",
+ dependencies=[dependency(root, "second")],
+ )
+ second = package(
+ root,
+ "second",
+ dependencies=[dependency(root, "first")],
+ )
+ packages = [first, second]
+ metadata = {
+ "packages": packages,
+ "workspace_members": [package["id"] for package in packages],
+ }
+
+ with self.assertRaisesRegex(RuntimeError, "dependency cycle"):
+ plan_from_metadata(metadata, root)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/release_rust/test_publish.py b/.github/scripts/release_rust/test_publish.py
new file mode 100644
index 0000000..0fb7a4f
--- /dev/null
+++ b/.github/scripts/release_rust/test_publish.py
@@ -0,0 +1,147 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import subprocess
+import unittest
+from contextlib import contextmanager
+from pathlib import Path
+from unittest import mock
+
+from plan import Package
+from publish import already_published
+from publish import publish_package
+
+
+def subprocess_result(returncode: int, output: str):
+ return mock.Mock(returncode=returncode, stdout=output)
+
+
+class ReleaseRustPublishTest(unittest.TestCase):
+ def test_live_publish_fetches_a_new_token_for_every_attempt(self):
+ package = Package("reqsign-test", "1.0.0", "test")
+ tokens = iter(("first-token", "second-token"))
+ revoked: list[str] = []
+ cargo_tokens: list[str | None] = []
+ cargo_commands: list[list[str]] = []
+
+ @contextmanager
+ def token_provider():
+ token = next(tokens)
+ try:
+ yield token
+ finally:
+ revoked.append(token)
+
+ results = iter(
+ (
+ subprocess_result(
+ 1, "Too many requests. Please try again after invalid."
+ ),
+ subprocess_result(0, "published"),
+ )
+ )
+
+ def run(*args, **kwargs):
+ cargo_commands.append(args[0])
+ cargo_tokens.append(kwargs["env"].get("CARGO_REGISTRY_TOKEN"))
+ return next(results)
+
+ with (
+ mock.patch.dict(
+ "publish.os.environ",
+ {"CARGO_REGISTRY_TOKEN": "legacy-token"},
+ clear=True,
+ ),
+ mock.patch("publish.temporary_trusted_publishing_token", token_provider),
+ mock.patch("publish.subprocess.run", run),
+ mock.patch("publish.time.sleep") as sleep,
+ ):
+ result = publish_package(Path(), package)
+
+ self.assertEqual(result, "published")
+ self.assertEqual(
+ cargo_commands,
+ [
+ [
+ "cargo",
+ "publish",
+ "--package",
+ "reqsign-test",
+ "--no-verify",
+ ],
+ [
+ "cargo",
+ "publish",
+ "--package",
+ "reqsign-test",
+ "--no-verify",
+ ],
+ ],
+ )
+ self.assertEqual(cargo_tokens, ["first-token", "second-token"])
+ self.assertEqual(revoked, ["first-token", "second-token"])
+ sleep.assert_called_once_with(610)
+
+ def test_already_published_package_is_recoverable(self):
+ package = Package("reqsign-test", "1.0.0", "test")
+
+ @contextmanager
+ def token_provider():
+ yield "temporary-token"
+
+ with (
+ mock.patch("publish.temporary_trusted_publishing_token", token_provider),
+ mock.patch(
+ "publish.subprocess.run",
+ return_value=subprocess_result(
+ 101,
+ "error: crate reqsign-test@1.0.0 already exists on crates.io index",
+ ),
+ ),
+ ):
+ result = publish_package(Path(), package)
+
+ self.assertEqual(result, "already published")
+
+ def test_unrelated_already_exists_error_is_not_ignored(self):
+ package = Package("reqsign-test", "1.0.0", "test")
+ self.assertFalse(
+ already_published(
+ "crate another@1.0.0 already exists on crates.io index", package
+ )
+ )
+
+ def test_non_retryable_failure_is_reported(self):
+ package = Package("reqsign-test", "1.0.0", "test")
+
+ @contextmanager
+ def token_provider():
+ yield "temporary-token"
+
+ with (
+ mock.patch("publish.temporary_trusted_publishing_token", token_provider),
+ mock.patch(
+ "publish.subprocess.run",
+ return_value=subprocess_result(101, "permission denied"),
+ ),
+ self.assertRaises(subprocess.CalledProcessError),
+ ):
+ publish_package(Path(), package)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/release_rust/test_trusted_publishing.py b/.github/scripts/release_rust/test_trusted_publishing.py
new file mode 100644
index 0000000..4b982d3
--- /dev/null
+++ b/.github/scripts/release_rust/test_trusted_publishing.py
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import io
+import json
+import unittest
+from unittest import mock
+from urllib.parse import parse_qs
+from urllib.parse import urlsplit
+
+from trusted_publishing import _oidc_request_url
+from trusted_publishing import request_trusted_publishing_token
+from trusted_publishing import revoke_trusted_publishing_token
+from trusted_publishing import temporary_trusted_publishing_token
+
+
+class JsonResponse(io.BytesIO):
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+
+class TrustedPublishingTest(unittest.TestCase):
+ def test_oidc_request_url_preserves_existing_query(self):
+ result = _oidc_request_url(
+ "https://example.test/token?api-version=1", "crates.io"
+ )
+
+ query = parse_qs(urlsplit(result).query)
+ self.assertEqual(query["api-version"], ["1"])
+ self.assertEqual(query["audience"], ["crates.io"])
+
+ def test_token_exchange_uses_github_oidc_and_crates_io(self):
+ requests = []
+ responses = iter(
+ (
+ JsonResponse(b'{"value":"github-jwt"}'),
+ JsonResponse(b'{"token":"crates-token"}'),
+ )
+ )
+
+ def urlopen(request, timeout):
+ requests.append(request)
+ self.assertEqual(timeout, 30)
+ return next(responses)
+
+ environment = {
+ "ACTIONS_ID_TOKEN_REQUEST_URL": "https://github.test/oidc?api-version=1",
+ "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token",
+ }
+ with (
+ mock.patch.dict("trusted_publishing.os.environ", environment, clear=True),
+ mock.patch("trusted_publishing.urllib.request.urlopen", urlopen),
+ ):
+ token = request_trusted_publishing_token()
+
+ self.assertEqual(token, "crates-token")
+ self.assertEqual(requests[0].get_method(), "GET")
+ self.assertEqual(
+ parse_qs(urlsplit(requests[0].full_url).query)["audience"],
+ ["crates.io"],
+ )
+ self.assertEqual(
+ requests[0].get_header("Authorization"), "Bearer request-token"
+ )
+ self.assertEqual(requests[1].get_method(), "POST")
+ self.assertEqual(json.loads(requests[1].data), {"jwt": "github-jwt"})
+
+ def test_revoke_uses_the_temporary_token(self):
+ requests = []
+
+ def urlopen(request, timeout):
+ requests.append(request)
+ self.assertEqual(timeout, 30)
+ return JsonResponse(b"")
+
+ with mock.patch("trusted_publishing.urllib.request.urlopen", urlopen):
+ revoke_trusted_publishing_token("crates-token")
+
+ self.assertEqual(requests[0].get_method(), "DELETE")
+ self.assertEqual(requests[0].get_header("Authorization"), "Bearer crates-token")
+
+ def test_temporary_token_is_always_revoked(self):
+ with (
+ mock.patch(
+ "trusted_publishing.request_trusted_publishing_token",
+ return_value="temporary-token",
+ ),
+ mock.patch("trusted_publishing.revoke_trusted_publishing_token") as revoke,
+ ):
+ with self.assertRaisesRegex(RuntimeError, "publish failed"):
+ with temporary_trusted_publishing_token() as token:
+ self.assertEqual(token, "temporary-token")
+ raise RuntimeError("publish failed")
+
+ revoke.assert_called_once_with("temporary-token", "https://crates.io")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/release_rust/trusted_publishing.py b/.github/scripts/release_rust/trusted_publishing.py
new file mode 100644
index 0000000..c3cf51b
--- /dev/null
+++ b/.github/scripts/release_rust/trusted_publishing.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+
+DEFAULT_REGISTRY_URL = "https://crates.io"
+USER_AGENT = (
+ "apache-opendal-reqsign-release/1.0 (https://github.com/apache/opendal-reqsign)"
+)
+
+
+def _response_error(operation: str, error: urllib.error.HTTPError) -> RuntimeError:
+ try:
+ response = error.read().decode("utf-8", errors="replace")
+ finally:
+ error.close()
+ try:
+ details = json.loads(response)
+ errors = details.get("errors", [])
+ response = "; ".join(item["detail"] for item in errors)
+ except (json.JSONDecodeError, AttributeError, KeyError, TypeError):
+ pass
+ return RuntimeError(f"{operation} failed with HTTP {error.code}: {response}")
+
+
+def _request_json(request: urllib.request.Request, operation: str) -> dict[str, object]:
+ try:
+ with urllib.request.urlopen(request, timeout=30) as response:
+ result = json.load(response)
+ except urllib.error.HTTPError as error:
+ raise _response_error(operation, error) from None
+ except urllib.error.URLError as error:
+ raise RuntimeError(f"{operation} failed: {error.reason}") from None
+ if not isinstance(result, dict):
+ raise RuntimeError(f"{operation} returned an invalid response")
+ return result
+
+
+def _oidc_request_url(request_url: str, audience: str) -> str:
+ parsed = urllib.parse.urlsplit(request_url)
+ query = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)
+ query.append(("audience", audience))
+ return urllib.parse.urlunsplit(
+ (
+ parsed.scheme,
+ parsed.netloc,
+ parsed.path,
+ urllib.parse.urlencode(query),
+ parsed.fragment,
+ )
+ )
+
+
+def _mask_secret(value: str) -> None:
+ if os.environ.get("GITHUB_ACTIONS") == "true":
+ print(f"::add-mask::{value}", flush=True)
+
+
+def request_trusted_publishing_token(
+ registry_url: str = DEFAULT_REGISTRY_URL,
+) -> str:
+ request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL")
+ request_token = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN")
+ if not request_url or not request_token:
+ raise RuntimeError(
+ "GitHub OIDC is unavailable; grant this job `id-token: write`"
+ )
+
+ audience = registry_url.rstrip("/")
+ for prefix in ("https://", "http://"):
+ if audience.startswith(prefix):
+ audience = audience.removeprefix(prefix)
+ break
+ oidc_request = urllib.request.Request(
+ _oidc_request_url(request_url, audience),
+ headers={
+ "Authorization": f"Bearer {request_token}",
+ "User-Agent": USER_AGENT,
+ },
+ )
+ oidc_response = _request_json(oidc_request, "requesting a GitHub OIDC token")
+ jwt = oidc_response.get("value")
+ if not isinstance(jwt, str) or not jwt:
+ raise RuntimeError("GitHub OIDC response did not contain a token")
+ _mask_secret(jwt)
+
+ token_request = urllib.request.Request(
+ f"{registry_url.rstrip('/')}/api/v1/trusted_publishing/tokens",
+ data=json.dumps({"jwt": jwt}).encode(),
+ headers={
+ "Content-Type": "application/json",
+ "User-Agent": USER_AGENT,
+ },
+ method="POST",
+ )
+ token_response = _request_json(
+ token_request, "exchanging a crates.io Trusted Publishing token"
+ )
+ token = token_response.get("token")
+ if not isinstance(token, str) or not token:
+ raise RuntimeError(
+ "crates.io Trusted Publishing response did not contain a token"
+ )
+ _mask_secret(token)
+ return token
+
+
+def revoke_trusted_publishing_token(
+ token: str, registry_url: str = DEFAULT_REGISTRY_URL
+) -> None:
+ request = urllib.request.Request(
+ f"{registry_url.rstrip('/')}/api/v1/trusted_publishing/tokens",
+ headers={
+ "Authorization": f"Bearer {token}",
+ "User-Agent": USER_AGENT,
+ },
+ method="DELETE",
+ )
+ try:
+ with urllib.request.urlopen(request, timeout=30):
+ return
+ except urllib.error.HTTPError as error:
+ raise _response_error(
+ "revoking a crates.io Trusted Publishing token", error
+ ) from None
+ except urllib.error.URLError as error:
+ raise RuntimeError(
+ f"revoking a crates.io Trusted Publishing token failed: {error.reason}"
+ ) from None
+
+
+@contextmanager
+def temporary_trusted_publishing_token(
+ registry_url: str = DEFAULT_REGISTRY_URL,
+) -> Iterator[str]:
+ token = request_trusted_publishing_token(registry_url)
+ try:
+ yield token
+ finally:
+ revoke_trusted_publishing_token(token, registry_url)
diff --git a/.github/workflows/bootstrap_rust_crates.yml b/.github/workflows/bootstrap_rust_crates.yml
new file mode 100644
index 0000000..01025ab
--- /dev/null
+++ b/.github/workflows/bootstrap_rust_crates.yml
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Bootstrap Rust Crates
+run-name: Bootstrap Rust crates at ${{ github.sha }}
+
+on:
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: bootstrap-rust-crates
+ cancel-in-progress: false
+
+jobs:
+ discover:
+ runs-on: ubuntu-24.04
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ ref: ${{ github.sha }}
+ - name: Validate main dispatch
+ run: |
+ set -euo pipefail
+ if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then
+ echo "Bootstrap Rust Crates must be dispatched from main" >&2
+ exit 1
+ fi
+ test "$(git rev-parse HEAD)" = "${GITHUB_SHA}"
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+ with:
+ python-version: "3.11"
+ - name: Discover Rust crates
+ run: python3 .github/scripts/release_rust/bootstrap.py discover
+
+ bootstrap:
+ needs: discover
+ runs-on: ubuntu-24.04
+ environment: rust-bootstrap
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ ref: ${{ github.sha }}
+ - name: Validate main dispatch
+ run: |
+ set -euo pipefail
+ if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then
+ echo "Bootstrap Rust Crates must be dispatched from main" >&2
+ exit 1
+ fi
+ test "$(git rev-parse HEAD)" = "${GITHUB_SHA}"
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+ with:
+ python-version: "3.11"
+ - name: Audit and bootstrap crates.io packages
+ env:
+ CARGO_REGISTRY_BOOTSTRAP_TOKEN: ${{ secrets.CARGO_REGISTRY_BOOTSTRAP_TOKEN }}
+ run: python3 .github/scripts/release_rust/bootstrap.py apply
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 317c6fe..39b51ba 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -22,19 +22,66 @@ on:
push:
# only for formal releases, not for release candidates
tags: ["v[0-9]+.[0-9]+.[0-9]+"]
+ pull_request:
+ branches:
+ - main
+ paths:
+ - ".github/scripts/release_rust/**"
+ - ".github/workflows/bootstrap_rust_crates.yml"
+ - ".github/workflows/release.yml"
+ - ".asf.yaml"
+ - "Cargo.toml"
+ - "**/Cargo.toml"
+ - "release/scripts/bootstrap-rust-crates.sh"
env:
CARGO_TERM_COLOR: always
- CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-permissions: {}
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: false
jobs:
+ validate:
+ name: Validate Publish Plan
+ runs-on: ubuntu-24.04
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+ with:
+ python-version: "3.11"
+ - name: Test release helpers
+ run: python3 -m unittest discover -s .github/scripts/release_rust -p "test_*.py"
+ - name: Check bootstrap helper syntax
+ run: bash -n release/scripts/bootstrap-rust-crates.sh
+ - name: Show publish plan
+ run: python3 .github/scripts/release_rust/plan.py
+ - name: Verify workspace packages
+ run: cargo publish --workspace --dry-run
+
publish:
name: Publish Workspace
+ if: >-
+ github.event_name == 'push' &&
+ startsWith(github.ref, 'refs/tags/') &&
+ !contains(github.ref, '-')
+ needs: validate
runs-on: ubuntu-24.04
+ environment: release
+ permissions:
+ contents: read
+ id-token: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - run: cargo publish --workspace
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+ with:
+ python-version: "3.11"
+ - name: Publish Rust crates
+ run: python3 .github/scripts/release_rust/publish.py
diff --git a/.gitignore b/.gitignore
index 1da474f..39e398d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
/target
Cargo.lock
.env
+__pycache__/
.claude/*.local.json
.idea
diff --git a/release/SKILL.md b/release/SKILL.md
index 13f9a76..9f51b17 100644
--- a/release/SKILL.md
+++ b/release/SKILL.md
@@ -12,7 +12,8 @@ Use this skill when preparing or executing an Apache OpenDAL reqsign release.
- Do not push the formal `vX.Y.Z` tag before the Apache vote passes.
- The voted release candidate tag is `vX.Y.Z-rc.N`; it must be a signed tag.
- The formal `vX.Y.Z` tag must point to the exact same commit as the voted RC tag, even if `main` has advanced after the vote started.
-- The repo release workflow is only for formal `vX.Y.Z` tags. It runs `cargo publish --workspace`.
+- Every crate in the current publish plan must exist with the exact `apache/opendal-reqsign`, `release.yml`, `release` Trusted Publisher and `trustpub_only` enabled before creating the RC tag.
+- The repo release workflow publishes only formal `vX.Y.Z` tags. It uses short-lived GitHub OIDC credentials and never a long-lived crates.io token.
- Source release artifacts live under Apache dist:
- RC: `https://dist.apache.org/repos/dist/dev/opendal/reqsign-X.Y.Z/`
- Final: `https://dist.apache.org/repos/dist/release/opendal/reqsign-X.Y.Z/`
@@ -40,14 +41,79 @@ Use this skill when preparing or executing an Apache OpenDAL reqsign release.
- Other workspace crates published in the same workspace release: patch bump.
- Keep root `[workspace.dependencies]` version requirements aligned with each crate manifest.
-3. Merge the version bump PR into `main`.
+3. Validate the publish plan and packages.
+
+ ```bash
+ python3 -m unittest discover \
+ -s .github/scripts/release_rust \
+ -p "test_*.py"
+ python3 .github/scripts/release_rust/plan.py
+ cargo publish --workspace --dry-run --allow-dirty
+ ```
+
+4. Merge the version bump PR into `main`.
Do not create the RC tag from an unmerged side branch unless the release manager explicitly accepts that the voted commit will not be on `main`.
+## Bootstrap Rust Crates
+
+Wait until the intended crate-name set is present on the
+`apache/opendal-reqsign` `main` branch. The release manager chooses the exact
+reservation time, normally about three days before the planned release. At that
+time, check out the current `main` commit and run:
+
+```bash
+release/scripts/bootstrap-rust-crates.sh
+```
+
+This command performs the transition to a bootstrapped publish plan. Run it
+without asking for a second confirmation after a PMC release manager explicitly
+chooses the reservation time. Do not run it for release status checks or dry-run
+planning. Publishing a `0.0.0` package is externally visible and irreversible,
+but it is a namespace reservation rather than an ASF software release.
+
+The helper:
+
+- Requires a clean checkout at the current `apache/opendal-reqsign` `main`.
+- Verifies that the `rust-bootstrap` environment has required reviewers.
+- Dispatches the input-free `bootstrap_rust_crates.yml` workflow.
+- Resolves the exact run, verifies its `headSha`, and waits for completion.
+- Blocks while the protected environment awaits PMC approval.
+- Verifies publicly that every crate in the checked publish plan exists and has
+ `trustpub_only` enabled.
+
+The protected job authenticates ownership and audits the complete publish plan
+before any write. Established crates must already have exactly one Trusted
+Publisher with:
+
+- Repository owner: `apache`
+- Repository name: `opendal-reqsign`
+- Workflow filename: `release.yml`
+- Environment: `release`
+
+The job never changes an established crate. Migrate those crates through
+crates.io before using the bootstrap workflow and enable `trustpub_only` on
+each. For a missing name, the workflow publishes a dependency-free `0.0.0`
+placeholder, creates the expected Trusted Publisher, enables `trustpub_only`,
+and reconciles partial placeholders on rerun.
+
+ASF Infrastructure provisions the `release` and `rust-bootstrap` environments
+from `.asf.yaml`. A PMC release manager must add
+`CARGO_REGISTRY_BOOTSTRAP_TOKEN` to the `rust-bootstrap` environment. The token
+must have only the `publish-new` and `trusted-publishing` endpoint scopes, with
+crate scopes restricted to reqsign package names. Never expose it to the normal
+release workflow.
+
+Run the helper again if the publish plan gains another crate after a successful
+bootstrap.
+
## Create RC Tag
Create the RC tag from the merged version-bump commit on `main`.
+Before tagging, confirm that the bootstrap workflow succeeded for the current
+publish plan and that no later commit added another crate.
+
```bash
git fetch origin main --tags
git switch --detach origin/main
@@ -233,6 +299,11 @@ NAME
gh run view RUN_ID --repo apache/opendal-reqsign --json status,conclusion,url,jobs
```
+ The workflow validates the complete workspace package set, then publishes
+ crates in dependency order. Every package attempt receives a new
+ OIDC-derived crates.io token that is revoked immediately afterward. Reruns
+ skip versions already published by an earlier partial run.
+
6. Verify crates.io versions.
```bash
@@ -255,3 +326,5 @@ NAME
- If `main` advances after the vote starts, formal `vX.Y.Z` still points to the RC commit, not latest `origin/main`.
- If SVN authentication fails, retry with `--force-interactive`; run `svn cleanup` if a killed commit leaves the working copy locked.
+- If crates.io publishing fails after some packages succeed, rerun the exact formal-tag workflow. Do not create another tag or use a long-lived token; the publisher resumes after matching already-published versions.
+- If the bootstrap workflow fails, inspect the authenticated preflight first. It deliberately refuses to modify established crates or continue with unexpected repository or Trusted Publisher metadata.
diff --git a/release/scripts/bootstrap-rust-crates.sh b/release/scripts/bootstrap-rust-crates.sh
new file mode 100755
index 0000000..026f953
--- /dev/null
+++ b/release/scripts/bootstrap-rust-crates.sh
@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo_dir="$(git -C "${script_dir}" rev-parse --show-toplevel)"
+cd "${repo_dir}"
+
+for command in gh git jq python3; do
+ command -v "${command}" >/dev/null || {
+ echo "required command is unavailable: ${command}" >&2
+ exit 1
+ }
+done
+
+if [[ "$#" -ne 0 ]]; then
+ echo "usage: $0" >&2
+ exit 1
+fi
+if [[ -n "$(git status --porcelain)" ]]; then
+ echo "the release checkout must be clean" >&2
+ exit 1
+fi
+
+apache_remote="$(
+ git remote -v |
+ awk '$2 ~ /github.com[:\/]apache\/opendal-reqsign(\.git)?$/ && $3 == "(fetch)" { print $1; exit }'
+)"
+if [[ -z "${apache_remote}" ]]; then
+ echo "cannot find a git remote for apache/opendal-reqsign" >&2
+ exit 1
+fi
+
+git fetch "${apache_remote}" main
+source_commit="$(git rev-parse FETCH_HEAD)"
+if [[ "$(git rev-parse HEAD)" != "${source_commit}" ]]; then
+ echo "the release checkout must be at the current apache/opendal-reqsign main: ${source_commit}" >&2
+ exit 1
+fi
+
+git cat-file -e \
+ "${source_commit}:.github/workflows/bootstrap_rust_crates.yml"
+git cat-file -e \
+ "${source_commit}:.github/scripts/release_rust/bootstrap.py"
+
+repo="apache/opendal-reqsign"
+workflow="bootstrap_rust_crates.yml"
+environment="rust-bootstrap"
+if ! environment_json="$(gh api "repos/${repo}/environments/${environment}")"; then
+ echo "GitHub environment ${environment} is not configured" >&2
+ exit 1
+fi
+if ! jq -e \
+ '[.protection_rules[]? | select(.type == "required_reviewers")] | length > 0' \
+ >/dev/null <<<"${environment_json}"; then
+ echo "GitHub environment ${environment} must require reviewers" >&2
+ exit 1
+fi
+
+run_title="Bootstrap Rust crates at ${source_commit}"
+started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+if ! dispatch_output="$(
+ gh workflow run "${workflow}" \
+ --repo "${repo}" \
+ --ref main 2>&1
+)"; then
+ echo "${dispatch_output}" >&2
+ exit 1
+fi
+echo "${dispatch_output}"
+
+run_id="$(
+ sed -nE \
+ 's#.*github\.com/apache/opendal-reqsign/actions/runs/([0-9]+).*#\1#p' \
+ <<<"${dispatch_output}" |
+ tail -n 1
+)"
+deadline=$((SECONDS + 120))
+while [[ -z "${run_id}" && ${SECONDS} -lt ${deadline} ]]; do
+ runs="$(
+ gh run list \
+ --repo "${repo}" \
+ --workflow "${workflow}" \
+ --event workflow_dispatch \
+ --limit 50 \
+ --json databaseId,displayTitle,headSha,createdAt
+ )"
+ run_id="$(
+ jq -r \
+ --arg title "${run_title}" \
+ --arg head_sha "${source_commit}" \
+ --arg started_at "${started_at}" \
+ '[.[]
+ | select(
+ .displayTitle == $title
+ and .headSha == $head_sha
+ and .createdAt >= $started_at
+ )
+ ]
+ | sort_by(.createdAt)
+ | last
+ | .databaseId // empty' \
+ <<<"${runs}"
+ )"
+ if [[ -z "${run_id}" ]]; then
+ sleep 2
+ fi
+done
+
+if [[ -z "${run_id}" ]]; then
+ echo "could not resolve the dispatched ${workflow} run" >&2
+ exit 1
+fi
+
+run_head_sha="$(
+ gh run view "${run_id}" --repo "${repo}" --json headSha --jq '.headSha'
+)"
+if [[ "${run_head_sha}" != "${source_commit}" ]]; then
+ echo "workflow run ${run_id} uses ${run_head_sha}, expected ${source_commit}" >&2
+ exit 1
+fi
+
+echo "Waiting for crates.io bootstrap run ${run_id}"
+if ! gh run watch "${run_id}" --repo "${repo}" --exit-status; then
+ gh run view "${run_id}" --repo "${repo}" --log-failed
+ exit 1
+fi
+
+gh run view "${run_id}" \
+ --repo "${repo}" \
+ --json displayTitle,headSha,status,conclusion,url \
+ --jq '{title: .displayTitle, head_sha: .headSha, status, conclusion, url}'
+
+python3 .github/scripts/release_rust/bootstrap.py verify