You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2026/09/17 06:31:01 UTC
(tsfile) 18/24: fix(ci): verify native package bundle integrity
This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch feat/native-package-artifacts
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 0bfaa0bc20990846fde2a20555307bd4e8bebb55
Author: ColinLee <sh...@163.com>
AuthorDate: Thu Sep 10 20:58:36 2026 +0800
fix(ci): verify native package bundle integrity
---
.github/workflows/native-packages.yml | 34 +-----
packaging/scripts/assemble_native_packages.py | 134 +++++++++++++++++++++--
packaging/tests/test_assemble_native_packages.py | 57 ++++++++++
3 files changed, 186 insertions(+), 39 deletions(-)
diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml
index dc01c872d..3c326d2e2 100644
--- a/.github/workflows/native-packages.yml
+++ b/.github/workflows/native-packages.yml
@@ -715,38 +715,8 @@ jobs:
LOGICAL_VERSION: ${{ needs.prepare.outputs.logical_version }}
shell: bash
run: |
- python3 - <<'PY'
- import hashlib
- import json
- import os
- import re
- from pathlib import Path
-
- output = Path(f"tsfile-native-packages-{os.environ['LOGICAL_VERSION']}")
- manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8"))
- checksums = (output / "SHA256SUMS").read_text(encoding="utf-8").splitlines()
- expected_checksums = []
- paths = set()
- for artifact in manifest["artifacts"]:
- relative = Path(artifact["path"])
- assert relative.as_posix() == artifact["path"]
- assert relative not in paths, f"Duplicate manifest path: {relative}"
- paths.add(relative)
- path = output / relative
- assert path.is_file() and not path.is_symlink(), f"Missing artifact: {relative}"
- digest = hashlib.sha256(path.read_bytes()).hexdigest()
- assert re.fullmatch(r"[0-9a-f]{64}", artifact["sha256"])
- assert digest == artifact["sha256"], f"Hash mismatch: {relative}"
- assert path.stat().st_size == artifact["size"], f"Size mismatch: {relative}"
- expected_checksums.append(f"{digest} {relative.as_posix()}")
- assert checksums == sorted(expected_checksums)
- actual_paths = {
- path.relative_to(output)
- for path in output.rglob("*")
- if path.is_file() and path.name not in {"SHA256SUMS", "manifest.json"}
- }
- assert actual_paths == paths
- PY
+ python3 packaging/scripts/assemble_native_packages.py \
+ --verify-bundle "tsfile-native-packages-$LOGICAL_VERSION"
- name: Upload checksummed native package bundle
uses: actions/upload-artifact@v7
diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py
index d9c6a356c..51cec3769 100644
--- a/packaging/scripts/assemble_native_packages.py
+++ b/packaging/scripts/assemble_native_packages.py
@@ -24,6 +24,7 @@ import argparse
import hashlib
import json
import os
+import re
import shutil
from pathlib import Path
from typing import Sequence
@@ -47,6 +48,27 @@ def _validate_metadata(values: dict[str, str], required: set[str], name: str) ->
raise ValueError(f"{name} must map strings to strings")
+def _validate_versions(versions: dict[str, str]) -> None:
+ _validate_metadata(versions, REQUIRED_VERSIONS, "versions")
+ for name in REQUIRED_VERSIONS:
+ value = versions[name]
+ if (
+ not value.strip()
+ or value in {".", ".."}
+ or "/" in value
+ or "\\" in value
+ or "\x00" in value
+ ):
+ raise ValueError(f"version {name} must be a safe path component")
+
+
+def _validate_source(source: dict[str, str]) -> None:
+ _validate_metadata(source, REQUIRED_SOURCE, "source")
+ for name in REQUIRED_SOURCE:
+ if not source[name].strip():
+ raise ValueError(f"source {name} must be nonempty")
+
+
def _input_files(input_dir: Path) -> list[Path]:
if not input_dir.is_dir() or input_dir.is_symlink():
raise ValueError(f"input directory is not a real directory: {input_dir}")
@@ -162,12 +184,80 @@ def _sha256(path: Path) -> str:
return digest.hexdigest()
+def _artifact_paths(output_dir: Path) -> set[Path]:
+ paths: set[Path] = set()
+ for path in output_dir.rglob("*"):
+ if path.is_symlink():
+ raise ValueError(f"symbolic link output is not allowed: {path}")
+ if path.is_file() and path.name not in {"SHA256SUMS", "manifest.json"}:
+ paths.add(path.relative_to(output_dir))
+ return paths
+
+
+def verify_bundle(output_dir: Path) -> None:
+ """Verify manifest hashes, sizes, and path-sorted checksums in a bundle."""
+ output_dir = Path(output_dir)
+ if not output_dir.is_dir() or output_dir.is_symlink():
+ raise ValueError(f"bundle directory is not a real directory: {output_dir}")
+ manifest_path = output_dir / "manifest.json"
+ checksums_path = output_dir / "SHA256SUMS"
+ if (
+ not manifest_path.is_file()
+ or manifest_path.is_symlink()
+ or not checksums_path.is_file()
+ or checksums_path.is_symlink()
+ ):
+ raise ValueError("bundle must contain real manifest.json and SHA256SUMS files")
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+ artifacts = manifest.get("artifacts") if isinstance(manifest, dict) else None
+ if not isinstance(artifacts, list):
+ raise ValueError("manifest artifacts must be a list")
+
+ paths: set[Path] = set()
+ for artifact in artifacts:
+ if not isinstance(artifact, dict):
+ raise ValueError("manifest artifacts must be objects")
+ relative_text = artifact.get("path")
+ if not isinstance(relative_text, str):
+ raise ValueError("manifest artifact path must be a string")
+ relative = Path(relative_text)
+ if (
+ relative.is_absolute()
+ or relative.as_posix() != relative_text
+ or ".." in relative.parts
+ ):
+ raise ValueError(f"unsafe manifest path: {relative_text}")
+ if relative in paths:
+ raise ValueError(f"duplicate manifest path: {relative_text}")
+ paths.add(relative)
+ path = output_dir / relative
+ if not path.is_file() or path.is_symlink():
+ raise ValueError(f"missing artifact: {relative_text}")
+ sha256 = artifact.get("sha256")
+ if not isinstance(sha256, str) or re.fullmatch(r"[0-9a-f]{64}", sha256) is None:
+ raise ValueError(f"invalid artifact SHA-256: {relative_text}")
+ if _sha256(path) != sha256:
+ raise ValueError(f"hash mismatch: {relative_text}")
+ if path.stat().st_size != artifact.get("size"):
+ raise ValueError(f"size mismatch: {relative_text}")
+
+ actual_checksums = checksums_path.read_text(encoding="utf-8").splitlines()
+ expected_checksums = [
+ f"{artifact['sha256']} {artifact['path']}"
+ for artifact in sorted(artifacts, key=lambda artifact: artifact["path"])
+ ]
+ if actual_checksums != expected_checksums:
+ raise ValueError("SHA256SUMS does not match path-sorted manifest artifacts")
+ if _artifact_paths(output_dir) != paths:
+ raise ValueError("bundle files do not match manifest artifacts")
+
+
def assemble(
input_dir: Path, output_dir: Path, versions: dict[str, str], source: dict[str, str]
) -> dict[str, object]:
"""Copy approved artifacts and return their deterministic publication manifest."""
- _validate_metadata(versions, REQUIRED_VERSIONS, "versions")
- _validate_metadata(source, REQUIRED_SOURCE, "source")
+ _validate_versions(versions)
+ _validate_source(source)
input_dir = Path(input_dir)
output_dir = Path(output_dir)
files = _input_files(input_dir)
@@ -219,17 +309,47 @@ def assemble(
def _argument_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--input-dir", type=Path, required=True)
- parser.add_argument("--output-dir", type=Path, required=True)
- parser.add_argument("--versions-json", type=Path, required=True)
- parser.add_argument("--source-repository", required=True)
- parser.add_argument("--source-commit", required=True)
+ parser.add_argument("--input-dir", type=Path)
+ parser.add_argument("--output-dir", type=Path)
+ parser.add_argument("--versions-json", type=Path)
+ parser.add_argument("--source-repository")
+ parser.add_argument("--source-commit")
+ parser.add_argument("--verify-bundle", type=Path)
return parser
def main(argv: Sequence[str] | None = None) -> int:
"""Assemble a bundle from an Actions artifact download directory."""
arguments = _argument_parser().parse_args(argv)
+ if arguments.verify_bundle is not None:
+ if any(
+ value is not None
+ for value in (
+ arguments.input_dir,
+ arguments.output_dir,
+ arguments.versions_json,
+ arguments.source_repository,
+ arguments.source_commit,
+ )
+ ):
+ raise ValueError(
+ "--verify-bundle cannot be combined with assembly arguments"
+ )
+ verify_bundle(arguments.verify_bundle)
+ return 0
+ missing = [
+ name
+ for name, value in (
+ ("--input-dir", arguments.input_dir),
+ ("--output-dir", arguments.output_dir),
+ ("--versions-json", arguments.versions_json),
+ ("--source-repository", arguments.source_repository),
+ ("--source-commit", arguments.source_commit),
+ )
+ if value is None
+ ]
+ if missing:
+ raise ValueError(f"missing assembly arguments: {', '.join(missing)}")
versions = json.loads(arguments.versions_json.read_text(encoding="utf-8"))
assemble(
arguments.input_dir,
diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py
index ab2342378..54da32c66 100644
--- a/packaging/tests/test_assemble_native_packages.py
+++ b/packaging/tests/test_assemble_native_packages.py
@@ -195,6 +195,63 @@ class AssembleNativePackagesTest(unittest.TestCase):
],
)
+ def test_verifies_assembler_checksum_lines_sorted_by_path(self):
+ module = self.require_module()
+
+ with tempfile.TemporaryDirectory() as temporary_directory:
+ input_directory = Path(temporary_directory) / "input"
+ output_directory = Path(temporary_directory) / "bundle"
+ self.write_fixture(input_directory)
+ module.assemble(input_directory, output_directory, VERSIONS, SOURCE)
+
+ checksum_lines = (
+ (output_directory / "SHA256SUMS")
+ .read_text(encoding="utf-8")
+ .splitlines()
+ )
+ self.assertNotEqual(
+ checksum_lines,
+ sorted(checksum_lines),
+ "fixture must distinguish path ordering from hash ordering",
+ )
+ self.assertIsNone(module.verify_bundle(output_directory))
+
+ def test_rejects_unsafe_versions_and_empty_source_identity(self):
+ module = self.require_module()
+
+ with tempfile.TemporaryDirectory() as temporary_directory:
+ input_directory = Path(temporary_directory) / "input"
+ self.write_fixture(input_directory)
+ for index, (field, value) in enumerate(
+ (
+ ("archive_version", "../../latest"),
+ ("homebrew_version", "../latest"),
+ ("archive_version", ""),
+ ("homebrew_version", "version\\latest"),
+ )
+ ):
+ with self.subTest(field=field, value=value):
+ versions = {**VERSIONS, field: value}
+ with self.assertRaisesRegex(ValueError, "safe path component"):
+ module.assemble(
+ input_directory,
+ Path(temporary_directory) / f"invalid-{index}",
+ versions,
+ SOURCE,
+ )
+ for index, (field, value) in enumerate(
+ (("commit", ""), ("repository", " "))
+ ):
+ with self.subTest(field=field, value=value):
+ source = {**SOURCE, field: value}
+ with self.assertRaisesRegex(ValueError, "nonempty"):
+ module.assemble(
+ input_directory,
+ Path(temporary_directory) / f"empty-{index}",
+ VERSIONS,
+ source,
+ )
+
def test_rejects_unknown_files(self):
module = self.require_module()