You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/09/06 17:02:29 UTC
(opendal) 01/02: fix(dev): make source release archives reproducible
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/reproducible-source-archives
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 840bf761292c1cc999dbe61631c5683a73574f8e
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Sep 7 01:01:44 2026 +0800
fix(dev): make source release archives reproducible
---
.github/workflows/ci_odev.yml | 50 ++++++
dev/src/main.rs | 11 +-
dev/src/release/mod.rs | 191 +++++++++++++++++++++--
scripts/verify_source_reproducibility.py | 156 ++++++++++++++++++
website/community/release/release.md | 3 +
website/community/release/reproducible-source.md | 70 +++++++++
6 files changed, 462 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/ci_odev.yml b/.github/workflows/ci_odev.yml
index b956ea01e0..d79f5d64c4 100644
--- a/.github/workflows/ci_odev.yml
+++ b/.github/workflows/ci_odev.yml
@@ -27,6 +27,9 @@ on:
paths:
- "**/*.rs"
- ".github/workflows/ci_odev.yml"
+ - "dev/Cargo.toml"
+ - "dev/Cargo.lock"
+ - "scripts/verify_source_reproducibility.py"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
@@ -92,3 +95,50 @@ jobs:
- name: Check diff
run: git diff --exit-code
+
+ reproduce:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+ - uses: actions/setup-python@v6
+ with:
+ python-version: '3.12'
+ - uses: ./.github/actions/setup
+ - name: Rebuild source packages in independent checkouts
+ run: python3 scripts/verify_source_reproducibility.py --output "$RUNNER_TEMP/reproduction"
+ - uses: actions/upload-artifact@v7
+ if: always()
+ with:
+ name: source-reproducibility-${{ matrix.os }}
+ path: |
+ ${{ runner.temp }}/reproduction/report.json
+ ${{ runner.temp }}/reproduction/build-*.log
+ if-no-files-found: error
+
+ compare:
+ needs: reproduce
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/download-artifact@v8
+ with:
+ pattern: source-reproducibility-*
+ path: reproduction
+ - name: Compare Linux and macOS archive digests
+ run: |
+ python3 - <<'PYTHON'
+ import json
+ from pathlib import Path
+ reports = [json.loads(p.read_text()) for p in Path("reproduction").glob("*/report.json")]
+ assert len(reports) == 2, "missing platform report"
+ assert all(r["matched"] for r in reports), "checkout reproduction failed"
+ assert reports[0]["commit"] == reports[1]["commit"], "different source commits"
+ assert reports[0]["runs"][0] == reports[1]["runs"][0], "Linux and macOS artifacts differ"
+ print("All source archives match across Linux and macOS")
+ PYTHON
diff --git a/dev/src/main.rs b/dev/src/main.rs
index 7b65e4bb75..8459f19cf0 100644
--- a/dev/src/main.rs
+++ b/dev/src/main.rs
@@ -62,7 +62,13 @@ enum Commands {
/// Update the version of all packages.
UpdateVersion,
/// Create all the release artifacts.
- Release,
+ Release {
+ /// Build source archives without invoking GPG.
+ #[arg(long)]
+ unsigned: bool,
+ },
+ /// Print the source package inventory as JSON.
+ ReleasePackages,
}
fn main() -> anyhow::Result<()> {
@@ -71,6 +77,7 @@ fn main() -> anyhow::Result<()> {
match Cmd::parse().command {
Commands::Generate { language } => generate::run(&language),
Commands::UpdateVersion => release::update_version(),
- Commands::Release => release::archive_package(),
+ Commands::Release { unsigned } => release::archive_package(!unsigned),
+ Commands::ReleasePackages => release::print_packages(),
}
}
diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs
index 04463ad240..c07e191c32 100644
--- a/dev/src/release/mod.rs
+++ b/dev/src/release/mod.rs
@@ -21,6 +21,7 @@ use flate2::write::GzEncoder;
use sha2::{Digest, Sha512};
use std::io::BufReader;
use std::io::Read;
+use std::path::Path;
mod bump;
mod package;
@@ -39,7 +40,23 @@ pub fn update_version() -> anyhow::Result<()> {
Ok(())
}
-pub fn archive_package() -> anyhow::Result<()> {
+pub fn print_packages() -> anyhow::Result<()> {
+ let packages = package::all_packages();
+ let inventory = packages
+ .iter()
+ .map(|p| {
+ serde_json::json!({
+ "path": p.name(), "version": p.version().to_string(),
+ "dependencies": p.dependencies().iter().map(|d| d.name()).collect::<Vec<_>>(),
+ "public_dependencies": p.public_compat_dependencies(),
+ })
+ })
+ .collect::<Vec<_>>();
+ println!("{}", serde_json::to_string_pretty(&inventory)?);
+ Ok(())
+}
+
+pub fn archive_package(sign: bool) -> anyhow::Result<()> {
std::fs::create_dir_all(workspace_dir().join("dist"))?;
let workspace_dir = workspace_dir();
@@ -48,16 +65,20 @@ pub fn archive_package() -> anyhow::Result<()> {
let packages = package::all_packages();
for package in packages {
let mut cmd = find_command("git", &workspace_dir);
- cmd.args(["ls-files", "LICENSE", "NOTICE"]);
+ cmd.args(["ls-files", "--stage", "-z", "--", "LICENSE", "NOTICE"]);
cmd.arg(package.name());
for dep in package.dependencies() {
cmd.arg(dep.name());
}
- let output = cmd.output().expect("failed to execute git ls-files");
- let output = String::from_utf8_lossy(&output.stdout);
- let files = output.lines().collect::<Vec<_>>();
+ let output = cmd.output()?;
+ anyhow::ensure!(output.status.success(), "git ls-files failed");
+ let files = archive_entries(&output.stdout)?;
archive_and_checksum(&package, &files)?;
+ if !sign {
+ continue;
+ }
+
let filename = format!("{}.tar.gz", package.make_prefix());
println!("Generate signature for package: {}", package.name());
@@ -70,7 +91,7 @@ pub fn archive_package() -> anyhow::Result<()> {
"--detach-sign",
filename.as_str(),
]);
- cmd.output().expect("failed to sign the package");
+ anyhow::ensure!(cmd.status()?.success(), "failed to sign {filename}");
println!("Check signature for package: {}", package.name());
let mut cmd = find_command("gpg", &dist_dir);
@@ -79,28 +100,20 @@ pub fn archive_package() -> anyhow::Result<()> {
format!("{filename}.asc").as_str(),
filename.as_str(),
]);
- cmd.output()
- .expect("failed to verify the package signature");
+ anyhow::ensure!(cmd.status()?.success(), "failed to verify {filename}");
}
Ok(())
}
-fn archive_and_checksum(package: &package::Package, files: &[&str]) -> anyhow::Result<()> {
+fn archive_and_checksum(package: &package::Package, files: &[ArchiveEntry]) -> anyhow::Result<()> {
println!("Archiving package: {}", package.name());
let prefix = package.make_prefix();
let filename = format!("{prefix}.tar.gz");
let tarball = workspace_dir().join("dist").join(&filename);
- {
- let tarball = std::fs::File::create(&tarball)?;
- let encoder = GzEncoder::new(tarball, Compression::default());
- let mut tar = tar::Builder::new(encoder);
- for file in files {
- tar.append_path_with_name(workspace_dir().join(file), format!("{prefix}/{file}"))?;
- }
- }
+ write_archive(&workspace_dir(), &tarball, &prefix, files)?;
{
let tarball = std::fs::File::open(&tarball)?;
@@ -126,6 +139,150 @@ fn archive_and_checksum(package: &package::Package, files: &[&str]) -> anyhow::R
Ok(())
}
+struct ArchiveEntry {
+ path: String,
+ mode: u32,
+}
+
+fn archive_entries(index: &[u8]) -> anyhow::Result<Vec<ArchiveEntry>> {
+ let mut entries = Vec::new();
+ for entry in index
+ .split(|byte| *byte == 0)
+ .filter(|entry| !entry.is_empty())
+ {
+ let entry = std::str::from_utf8(entry)?;
+ let (metadata, path) = entry
+ .split_once('\t')
+ .ok_or_else(|| anyhow::anyhow!("invalid Git index entry"))?;
+ let fields = metadata.split_whitespace().collect::<Vec<_>>();
+ anyhow::ensure!(
+ fields.len() == 3 && fields[2] == "0",
+ "unmerged Git index entry: {path}"
+ );
+ let mode = u32::from_str_radix(fields[0], 8)?;
+ anyhow::ensure!(
+ matches!(mode, 0o100644 | 0o100755 | 0o120000),
+ "unsupported Git mode for {path}"
+ );
+ entries.push(ArchiveEntry {
+ path: path.to_owned(),
+ mode,
+ });
+ }
+ entries.sort_by(|a, b| a.path.cmp(&b.path));
+ Ok(entries)
+}
+
+fn write_archive(
+ root: &Path,
+ output: &Path,
+ prefix: &str,
+ files: &[ArchiveEntry],
+) -> anyhow::Result<()> {
+ let encoder = GzEncoder::new(std::fs::File::create(output)?, Compression::default());
+ let mut tar = tar::Builder::new(encoder);
+ for entry in files {
+ // Git owns executable bits; checkout timestamps, owners and umask must not
+ // affect the release bytes. Gzip's default header has no timestamp/name.
+ let mut header = tar::Header::new_gnu();
+ header.set_uid(0);
+ header.set_gid(0);
+ header.set_mtime(0);
+ let path = root.join(&entry.path);
+ let archive_path = format!("{prefix}/{}", entry.path);
+ if entry.mode == 0o120000 {
+ header.set_mode(0o777);
+ header.set_entry_type(tar::EntryType::Symlink);
+ header.set_size(0);
+ tar.append_link(&mut header, archive_path, std::fs::read_link(path)?)?;
+ } else {
+ header.set_mode(entry.mode & 0o777);
+ header.set_entry_type(tar::EntryType::Regular);
+ let file = std::fs::File::open(path)?;
+ header.set_size(file.metadata()?.len());
+ tar.append_data(&mut header, archive_path, file)?;
+ }
+ }
+ tar.into_inner()?.finish()?;
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::fs::FileTimes;
+ use std::time::{Duration, UNIX_EPOCH};
+
+ #[test]
+ fn archive_ignores_checkout_metadata() -> anyhow::Result<()> {
+ let root = std::env::temp_dir().join(format!("opendal-archive-{}", std::process::id()));
+ std::fs::create_dir_all(&root)?;
+ let file = root.join("script");
+ std::fs::write(&file, b"#!/bin/sh\necho reproducible\n")?;
+ let entries = archive_entries(b"100755 abc 0\tscript\0")?;
+ for (name, seconds) in [("a.tar.gz", 1_700_000_000), ("b.tar.gz", 1_800_000_000)] {
+ std::fs::File::open(&file)?.set_times(
+ FileTimes::new().set_modified(UNIX_EPOCH + Duration::from_secs(seconds)),
+ )?;
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt;
+ std::fs::set_permissions(
+ &file,
+ std::fs::Permissions::from_mode(if name.starts_with('a') {
+ 0o600
+ } else {
+ 0o755
+ }),
+ )?;
+ }
+ write_archive(&root, &root.join(name), "source", &entries)?;
+ }
+ assert_eq!(
+ std::fs::read(root.join("a.tar.gz"))?,
+ std::fs::read(root.join("b.tar.gz"))?
+ );
+ let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(std::fs::File::open(
+ root.join("a.tar.gz"),
+ )?));
+ let entry = archive.entries()?.next().unwrap()?;
+ assert_eq!(entry.header().mode()?, 0o755);
+ assert_eq!(entry.header().mtime()?, 0);
+ assert_eq!(entry.header().uid()?, 0);
+ assert_eq!(entry.header().gid()?, 0);
+ std::fs::remove_dir_all(root)?;
+ Ok(())
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn archive_preserves_symlinks_without_following_them() -> anyhow::Result<()> {
+ let root = std::env::temp_dir().join(format!("opendal-symlink-{}", std::process::id()));
+ std::fs::create_dir_all(&root)?;
+ std::os::unix::fs::symlink("missing-target", root.join("link"))?;
+ let entries = archive_entries(b"120000 abc 0\tlink\0")?;
+ write_archive(&root, &root.join("source.tar.gz"), "source", &entries)?;
+ let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(std::fs::File::open(
+ root.join("source.tar.gz"),
+ )?));
+ let entry = archive.entries()?.next().unwrap()?;
+ assert!(entry.header().entry_type().is_symlink());
+ assert_eq!(entry.link_name()?.unwrap(), Path::new("missing-target"));
+ assert_eq!(entry.size(), 0);
+ std::fs::remove_dir_all(root)?;
+ Ok(())
+ }
+
+ #[test]
+ fn index_paths_are_sorted_and_conflicts_rejected() -> anyhow::Result<()> {
+ let entries = archive_entries(b"100644 a 0\tz\x00100644 b 0\ta\tname\0")?;
+ assert_eq!(entries[0].path, "a\tname");
+ assert!(archive_entries(b"100644 a 1\tconflict\0").is_err());
+ assert!(archive_entries(b"160000 a 0\tsubmodule\0").is_err());
+ Ok(())
+ }
+}
+
fn format_digest_hex(digest: impl AsRef<[u8]>) -> String {
use std::fmt::Write;
diff --git a/scripts/verify_source_reproducibility.py b/scripts/verify_source_reproducibility.py
new file mode 100644
index 0000000000..e73cc94da3
--- /dev/null
+++ b/scripts/verify_source_reproducibility.py
@@ -0,0 +1,156 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Rebuild every source archive in two checkouts with different filesystem metadata."""
+
+import argparse
+import hashlib
+import json
+import os
+import platform
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+
+def run(*args, **kwargs):
+ return subprocess.run([str(a) for a in args], check=True, **kwargs)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--revision", default="HEAD")
+ parser.add_argument(
+ "--working-tree",
+ action="store_true",
+ help="Include tracked local changes in both checkouts",
+ )
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument(
+ "--compare", type=Path, help="Also compare with downloaded CI tar.gz artifacts"
+ )
+ args = parser.parse_args()
+ repo = Path(__file__).resolve().parents[1]
+ sha = subprocess.check_output(
+ ["git", "rev-parse", args.revision], cwd=repo, text=True
+ ).strip()
+ patch = (
+ subprocess.check_output(["git", "diff", "--binary", sha], cwd=repo)
+ if args.working_tree
+ else b""
+ )
+ args.output.mkdir(parents=True, exist_ok=False)
+ (args.output / "input.patch").write_bytes(patch)
+ report = {
+ "platform": platform.platform(),
+ "rustc": subprocess.check_output(["rustc", "-Vv"], text=True),
+ "commit": sha,
+ "patch_sha512": hashlib.sha512(patch).hexdigest(),
+ "runs": [],
+ "matched": False,
+ }
+ try:
+ with tempfile.TemporaryDirectory(prefix="opendal-reproduce-") as tmp:
+ target = Path(tmp) / "target"
+ for index in range(2):
+ checkout = Path(tmp) / f"checkout-{index}"
+ run(
+ "git",
+ "clone",
+ "--shared",
+ "--no-checkout",
+ "-c",
+ "core.autocrlf=false",
+ repo,
+ checkout,
+ )
+ run("git", "checkout", "--detach", sha, cwd=checkout)
+ if patch:
+ run("git", "apply", "--binary", "-", cwd=checkout, input=patch)
+ tracked = subprocess.check_output(
+ ["git", "ls-files", "-z"], cwd=checkout
+ ).split(b"\0")
+ for raw in filter(None, tracked):
+ file = checkout / os.fsdecode(raw)
+ if file.is_symlink() or not file.is_file():
+ continue
+ os.utime(file, (1_700_000_000 + index * 100_000_000,) * 2)
+ if os.name == "posix":
+ os.chmod(file, 0o644 if index == 0 else 0o755)
+ env = {
+ **os.environ,
+ "CARGO_TARGET_DIR": str(target),
+ "TZ": "UTC" if index == 0 else "Asia/Shanghai",
+ }
+ log_path = args.output / f"build-{index}.log"
+ with log_path.open("wb") as log:
+ run(
+ "cargo",
+ "run",
+ "--locked",
+ "--manifest-path",
+ checkout / "dev/Cargo.toml",
+ "--",
+ "release",
+ "--unsigned",
+ cwd=checkout,
+ env=env,
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ )
+ inventory = json.loads(
+ subprocess.check_output(
+ [str(target / "debug/odev"), "release-packages"], cwd=checkout
+ )
+ )
+ archives = sorted((checkout / "dist").glob("*.tar.gz"))
+ if len(archives) != len(inventory) or not archives:
+ raise ValueError("source archive inventory is incomplete")
+ hashes = {
+ file.name: hashlib.sha512(file.read_bytes()).hexdigest()
+ for file in archives
+ }
+ report["runs"].append(hashes)
+ if index == 0:
+ for file in archives:
+ shutil.copy2(file, args.output / file.name)
+ shutil.copy2(
+ Path(str(file) + ".sha512"),
+ args.output / (file.name + ".sha512"),
+ )
+ if index and hashes != report["runs"][0]:
+ raise ValueError("independent checkout artifacts differ")
+ if args.compare:
+ expected = {
+ file.name: hashlib.sha512(file.read_bytes()).hexdigest()
+ for file in args.compare.glob("*.tar.gz")
+ }
+ if expected != report["runs"][0]:
+ raise ValueError(
+ "downloaded CI artifacts differ or inventory does not match"
+ )
+ report["matched"] = True
+ finally:
+ (args.output / "report.json").write_text(json.dumps(report, indent=2) + "\n")
+ print(
+ f"Verified {len(report['runs'][0])} identical source archives. Evidence: {args.output}"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/website/community/release/release.md b/website/community/release/release.md
index ca9720667f..0633a16fa0 100644
--- a/website/community/release/release.md
+++ b/website/community/release/release.md
@@ -6,6 +6,9 @@ sidebar_position: 3
This document mainly introduces how the release manager
releases a new version of Apache OpenDAL™ in accordance with the Apache requirements.
+See [Reproducible source archives](./reproducible-source.md) for rebuilding and
+comparing the source packages before signing.
+
## Introduction
`Source Release` is the key point which Apache values, and is also necessary for an ASF release.
diff --git a/website/community/release/reproducible-source.md b/website/community/release/reproducible-source.md
new file mode 100644
index 0000000000..7944fd2ac0
--- /dev/null
+++ b/website/community/release/reproducible-source.md
@@ -0,0 +1,70 @@
+---
+title: Reproducible source archives
+sidebar_position: 5
+---
+
+# Reproducible source archives
+
+OpenDAL source release archives must reproduce byte for byte from the same Git
+revision and locked packaging dependencies. This contract covers the `.tar.gz`
+files generated by `odev release`, including every binding and integration source
+package. It does not claim reproducibility for Maven artifacts, wheels, native
+binaries, or OpenPGP signature files.
+
+## Archive inputs
+
+Build from a clean checkout of the exact candidate commit, with Git line-ending
+conversion disabled (`core.autocrlf=false`) and native symlink support. Run the
+packager with `cargo run --locked --manifest-path dev/Cargo.toml -- release --unsigned`.
+The packager reads tracked working-tree contents and the Git index's file modes;
+local source edits therefore change the release bytes. It does not regenerate
+source files as part of packaging.
+
+The source package inventory comes from `dev/src/release/package.rs`. Each archive
+includes the package, its declared source dependencies, and the root LICENSE and
+NOTICE files. Paths are sorted. Tar entries use timestamp zero, numeric owner and
+group zero, empty owner/group names, and Git's regular/executable file modes.
+Symlinks retain their link target and are not followed. Gzip includes neither a
+filename nor a timestamp. The checked-in `dev/Cargo.lock` fixes the compression
+implementation. Changing those dependencies can change archive bytes and requires
+repeating the reproduction checks.
+
+## Reproduce the complete package set
+
+From a trusted checkout containing the verification script:
+
+```shell
+python3 scripts/verify_source_reproducibility.py \
+ --revision <candidate-commit> \
+ --output /tmp/opendal-reproduction
+```
+
+The output directory must not already exist. The script creates two independent
+checkouts, changes their file timestamps, permissions and process timezones, and
+builds every source archive in both. It records the commit, packaging patch digest,
+host and compiler information, all archive hashes, and build logs. A mismatch or
+missing package fails the command. The first set of archives is retained alongside
+the report. Temporary checkouts are removed automatically.
+
+For local packaging-tool development, `--working-tree` applies tracked changes to
+both checkouts and retains the exact input patch in the output. Such a run is
+useful regression evidence, but release verification must use the committed
+candidate without this option.
+
+To validate a CI candidate on trusted hardware, download the complete voted-on
+`.tar.gz` set to a directory and add:
+
+```shell
+python3 scripts/verify_source_reproducibility.py \
+ --revision <candidate-commit> \
+ --compare /path/to/downloaded-candidate \
+ --output /tmp/opendal-candidate-reproduction
+```
+
+This compares the full filename set and every archive's SHA-512 against the local
+rebuilds. Verify the downloaded signatures separately and include both results in
+the release review. Reproducibility does not replace license review or testing.
+
+odev CI performs the same check on Linux and macOS and compares
+their manifests. A CI-to-CI comparison is regression coverage; it does not replace
+the required independent reproduction on trusted hardware before publication.