You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/02/18 11:11:14 UTC
(opendal) 02/04: Fix build
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch refactor-release
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit d66eb74060263bc97ad4d393261f459030c9c80b
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Sun Feb 18 18:24:26 2024 +0800
Fix build
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
scripts/README.md | 2 +-
scripts/constants.py | 57 ++++++++++++++++++++--
scripts/release.py | 133 ++++++++++++++++++++++++++++++++-------------------
scripts/verify.py | 4 +-
4 files changed, 141 insertions(+), 55 deletions(-)
diff --git a/scripts/README.md b/scripts/README.md
index 0d721e6488..cb3f1339ce 100644
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -8,7 +8,7 @@ NOTES: all scripts must be running at root folder of OpenDAL project.
## Release
```shell
-OPENDAL_VERSION=0.30.2 OPENDAL_VERSION_RC=rc1 ./scripts/release.sh
+./scripts/release.py
```
> Before running release, please make sure you have bump all versions.
diff --git a/scripts/constants.py b/scripts/constants.py
index 8e83cc2572..b952a379b7 100755
--- a/scripts/constants.py
+++ b/scripts/constants.py
@@ -17,7 +17,7 @@
# under the License.
from pathlib import Path
-
+import tomllib
ROOT_DIR = Path(__file__).parent.parent
@@ -34,6 +34,57 @@ def list_packages():
PACKAGES = list_packages()
+# package dependencies is used to maintain the dependencies between packages.
+#
+# All packages are depend on `core` by default, so we should only maintain the exceptions in list.
+PACKAGE_DEPENDENCIES = {
+ Path("bindings/go"): "bindings/c",
+ Path("bindings/swift"): "bindings/c",
+ Path("bindings/zig"): "bindings/c",
+}
+
+
+# fetch the package dependence, return `core` if not listed in `PACAKGE_DEPENDENCIES`.
+def get_package_dependence(package: Path) -> str:
+ return PACKAGE_DEPENDENCIES.get(package, "core")
+
+
+# input: Path to a Rust package like `core` and `bindings/python`.
+def get_rust_package_version(path):
+ with open(ROOT_DIR / path / "Cargo.toml", "rb") as f:
+ data = tomllib.load(f)
+ version = data["package"]["version"]
+ return version
+
+
+# get the package version by package name.
+#
+# For examples:
+# core: `0.45.0`
+# packages depends on core: `0.1.0+core.0.45.0`
+# packages depends on bindings/c: `0.1.0+bindings_c.0.1.0`
+def get_package_version(package):
+ if package == "core":
+ return get_rust_package_version("core")
+
+ # NOTE: for now, all dependence package must be rust package.
+ dependence = get_package_dependence(package)
+ dependence_version = get_rust_package_version(dependence)
+
+ cargo_toml = ROOT_DIR / package / "Cargo.toml"
+ # cargo_toml exists, we can get the version from Cargo.toml.
+ if cargo_toml.exists():
+ package_version = get_rust_package_version(package)
+ return f"{package_version}+{dependence.replace('/', '_')}.{dependence_version}"
+
+ # cargo_toml not exists, we should handle case by case ideally.
+ #
+ # However, those packages are not mature enough, it's much easier for us to always return `0.0.0` instead.
+ return f"0.0.0+{dependence.replace('/', '_')}.{dependence_version}"
+
+
if __name__ == "__main__":
- for project in PACKAGES:
- print(project)
+ for v in PACKAGES:
+ print(
+ f"{v}: depends={get_package_dependence(v)}, version={get_package_version(v)}"
+ )
diff --git a/scripts/release.py b/scripts/release.py
index 9bf1e86954..df02ea41b0 100755
--- a/scripts/release.py
+++ b/scripts/release.py
@@ -16,54 +16,89 @@
# specific language governing permissions and limitations
# under the License.
-import os
import subprocess
from pathlib import Path
-import tomllib
-
-OPENDAL_VERSION = os.getenv('OPENDAL_VERSION')
-OPENDAL_VERSION_RC = os.getenv('OPENDAL_VERSION_RC', 'rc.1')
-
-if not OPENDAL_VERSION:
- print("OPENDAL_VERSION is unset")
- exit(1)
-else:
- print(f"var is set to '{OPENDAL_VERSION}'")
-
-# tar source code
-release_version = OPENDAL_VERSION
-# rc versions
-rc_version = OPENDAL_VERSION_RC
-# Corresponding git repository branch
-git_branch = f"release-{release_version}-{rc_version}"
-
-dist_path = Path('dist')
-if not dist_path.exists():
- dist_path.mkdir()
-
-print("> Checkout version branch")
-subprocess.run(['git', 'checkout', '-B', git_branch], check=True)
-
-print("> Start package")
-subprocess.run(['git', 'archive', '--format=tar.gz', '--output', f"dist/apache-opendal-{release_version}-src.tar.gz", '--prefix', f"apache-opendal-{release_version}-src/", '--add-file=Cargo.toml', git_branch], check=True)
-
-os.chdir('dist')
-print("> Generate signature")
-for i in Path('.').glob('*.tar.gz'):
- print(i)
- subprocess.run(['gpg', '--armor', '--output', f"{i}.asc", '--detach-sig', str(i)], check=True)
-
-print("> Check signature")
-for i in Path('.').glob('*.tar.gz'):
- print(i)
- subprocess.run(['gpg', '--verify', f"{i}.asc", str(i)], check=True)
-
-print("> Generate sha512sum")
-for i in Path('.').glob('*.tar.gz'):
- print(i)
- subprocess.run(['sha512sum', str(i)], stdout=open(f"{i}.sha512", "w"))
-
-print("> Check sha512sum")
-for i in Path('.').glob('*.tar.gz.sha512'):
- print(i)
- subprocess.run(['sha512sum', '--check', str(i)], check=True)
+from constants import get_package_version, get_package_dependence, PACKAGES
+
+ROOT_DIR = Path(__file__).parent.parent
+
+# If this package depends on `core`, we need to update it's core version in `Cargo.toml` file.
+def update_package(path):
+ # skip core package.
+ if path == "core":
+ return
+ if get_package_dependence(path) != "core":
+ return
+
+ core_version = get_package_version("core")
+
+
+def archive_package(path):
+ print(f"Archive package {path} started")
+
+ version = get_package_version(path)
+ name = f"apache-opendal-{str(path).replace('/', '-')}-{version}-src"
+
+ ls_command = ["git", "ls-files", "."]
+ ls_result = subprocess.run(
+ ls_command, cwd=ROOT_DIR / path, capture_output=True, check=True, text=True
+ )
+
+ tar_command = [
+ "tar",
+ "-zcf",
+ f"{ROOT_DIR}/dist/{name}.tar.gz",
+ "--transform",
+ f"s,^,{name}/,",
+ "-T",
+ "-",
+ ]
+ subprocess.run(
+ tar_command,
+ cwd=ROOT_DIR / path,
+ input=ls_result.stdout.encode("utf-8"),
+ check=True,
+ )
+
+ print(f"Archive package {path} to dist/{name}.tar.gz")
+
+def generate_signature():
+ for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+ print(f"Generate signature for {i}")
+ subprocess.run(
+ ["gpg", "--yes", "--armor", "--output", f"{i}.asc", "--detach-sig", str(i)],
+ cwd=ROOT_DIR / "dist",
+ check=True,
+ )
+
+ for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+ print(f"Check signature for {i}")
+ subprocess.run(
+ ["gpg", "--verify", f"{i}.asc", str(i)], cwd=ROOT_DIR / "dist", check=True
+ )
+
+
+def generate_checksum():
+ for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+ print(f"Generate checksum for {i}")
+ subprocess.run(
+ ["sha512sum", str(i.relative_to(ROOT_DIR / "dist"))],
+ stdout=open(f"{i}.sha512", "w"),
+ cwd=ROOT_DIR / "dist",
+ check=True,
+ )
+
+ for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+ print(f"Check checksum for {i}")
+ subprocess.run(
+ ["sha512sum", "--check", f"{str(i.relative_to(ROOT_DIR / 'dist'))}.sha512"],
+ cwd=ROOT_DIR / "dist",
+ check=True,
+ )
+
+
+if __name__ == "__main__":
+ for v in PACKAGES:
+ archive_package(v)
+ generate_signature()
+ generate_checksum()
diff --git a/scripts/verify.py b/scripts/verify.py
index 81d6a41ba4..bfca20a520 100755
--- a/scripts/verify.py
+++ b/scripts/verify.py
@@ -47,7 +47,7 @@ def check_java():
def build_core():
print("Start building opendal core")
- subprocess.run(["cargo", "build", "--release"], check=True)
+ subprocess.run(["cargo", "build", "--release"], cwd="core", check=True)
def build_java_binding():
@@ -62,7 +62,7 @@ def build_java_binding():
"-Dcargo-build.profile=release",
],
check=True,
- cwd="bindings/java"
+ cwd="bindings/java",
)