You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/02/18 11:11:14 UTC

(opendal) 02/04: Fix build

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch refactor-release
in repository https://gitbox.apache.org/repos/asf/opendal.git

commit d66eb74060263bc97ad4d393261f459030c9c80b
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Sun Feb 18 18:24:26 2024 +0800

    Fix build
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 scripts/README.md    |   2 +-
 scripts/constants.py |  57 ++++++++++++++++++++--
 scripts/release.py   | 133 ++++++++++++++++++++++++++++++++-------------------
 scripts/verify.py    |   4 +-
 4 files changed, 141 insertions(+), 55 deletions(-)

diff --git a/scripts/README.md b/scripts/README.md
index 0d721e6488..cb3f1339ce 100644
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -8,7 +8,7 @@ NOTES: all scripts must be running at root folder of OpenDAL project.
 ## Release
 
 ```shell
-OPENDAL_VERSION=0.30.2 OPENDAL_VERSION_RC=rc1 ./scripts/release.sh
+./scripts/release.py
 ```
 
 > Before running release, please make sure you have bump all versions.
diff --git a/scripts/constants.py b/scripts/constants.py
index 8e83cc2572..b952a379b7 100755
--- a/scripts/constants.py
+++ b/scripts/constants.py
@@ -17,7 +17,7 @@
 # under the License.
 
 from pathlib import Path
-
+import tomllib
 
 ROOT_DIR = Path(__file__).parent.parent
 
@@ -34,6 +34,57 @@ def list_packages():
 
 PACKAGES = list_packages()
 
+# package dependencies is used to maintain the dependencies between packages.
+#
+# All packages are depend on `core` by default, so we should only maintain the exceptions in list.
+PACKAGE_DEPENDENCIES = {
+    Path("bindings/go"): "bindings/c",
+    Path("bindings/swift"): "bindings/c",
+    Path("bindings/zig"): "bindings/c",
+}
+
+
+# fetch the package dependence, return `core` if not listed in `PACAKGE_DEPENDENCIES`.
+def get_package_dependence(package: Path) -> str:
+    return PACKAGE_DEPENDENCIES.get(package, "core")
+
+
+# input: Path to a Rust package like `core` and `bindings/python`.
+def get_rust_package_version(path):
+    with open(ROOT_DIR / path / "Cargo.toml", "rb") as f:
+        data = tomllib.load(f)
+        version = data["package"]["version"]
+    return version
+
+
+# get the package version by package name.
+#
+# For examples:
+# core: `0.45.0`
+# packages depends on core: `0.1.0+core.0.45.0`
+# packages depends on bindings/c: `0.1.0+bindings_c.0.1.0`
+def get_package_version(package):
+    if package == "core":
+        return get_rust_package_version("core")
+
+    # NOTE: for now, all dependence package must be rust package.
+    dependence = get_package_dependence(package)
+    dependence_version = get_rust_package_version(dependence)
+
+    cargo_toml = ROOT_DIR / package / "Cargo.toml"
+    # cargo_toml exists, we can get the version from Cargo.toml.
+    if cargo_toml.exists():
+        package_version = get_rust_package_version(package)
+        return f"{package_version}+{dependence.replace('/', '_')}.{dependence_version}"
+
+    # cargo_toml not exists, we should handle case by case ideally.
+    #
+    # However, those packages are not mature enough, it's much easier for us to always return `0.0.0` instead.
+    return f"0.0.0+{dependence.replace('/', '_')}.{dependence_version}"
+
+
 if __name__ == "__main__":
-    for project in PACKAGES:
-        print(project)
+    for v in PACKAGES:
+        print(
+            f"{v}: depends={get_package_dependence(v)}, version={get_package_version(v)}"
+        )
diff --git a/scripts/release.py b/scripts/release.py
index 9bf1e86954..df02ea41b0 100755
--- a/scripts/release.py
+++ b/scripts/release.py
@@ -16,54 +16,89 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import os
 import subprocess
 from pathlib import Path
-import tomllib
-
-OPENDAL_VERSION = os.getenv('OPENDAL_VERSION')
-OPENDAL_VERSION_RC = os.getenv('OPENDAL_VERSION_RC', 'rc.1')
-
-if not OPENDAL_VERSION:
-    print("OPENDAL_VERSION is unset")
-    exit(1)
-else:
-    print(f"var is set to '{OPENDAL_VERSION}'")
-
-# tar source code
-release_version = OPENDAL_VERSION
-# rc versions
-rc_version = OPENDAL_VERSION_RC
-# Corresponding git repository branch
-git_branch = f"release-{release_version}-{rc_version}"
-
-dist_path = Path('dist')
-if not dist_path.exists():
-    dist_path.mkdir()
-
-print("> Checkout version branch")
-subprocess.run(['git', 'checkout', '-B', git_branch], check=True)
-
-print("> Start package")
-subprocess.run(['git', 'archive', '--format=tar.gz', '--output', f"dist/apache-opendal-{release_version}-src.tar.gz", '--prefix', f"apache-opendal-{release_version}-src/", '--add-file=Cargo.toml', git_branch], check=True)
-
-os.chdir('dist')
-print("> Generate signature")
-for i in Path('.').glob('*.tar.gz'):
-    print(i)
-    subprocess.run(['gpg', '--armor', '--output', f"{i}.asc", '--detach-sig', str(i)], check=True)
-
-print("> Check signature")
-for i in Path('.').glob('*.tar.gz'):
-    print(i)
-    subprocess.run(['gpg', '--verify', f"{i}.asc", str(i)], check=True)
-
-print("> Generate sha512sum")
-for i in Path('.').glob('*.tar.gz'):
-    print(i)
-    subprocess.run(['sha512sum', str(i)], stdout=open(f"{i}.sha512", "w"))
-
-print("> Check sha512sum")
-for i in Path('.').glob('*.tar.gz.sha512'):
-    print(i)
-    subprocess.run(['sha512sum', '--check', str(i)], check=True)
+from constants import get_package_version, get_package_dependence, PACKAGES
+
+ROOT_DIR = Path(__file__).parent.parent
+
+# If this package depends on `core`, we need to update it's core version in `Cargo.toml` file.
+def update_package(path):
+    # skip core package.
+    if path == "core":
+        return
+    if get_package_dependence(path) != "core":
+        return
+
+    core_version = get_package_version("core")
+
+
+def archive_package(path):
+    print(f"Archive package {path} started")
+
+    version = get_package_version(path)
+    name = f"apache-opendal-{str(path).replace('/', '-')}-{version}-src"
+
+    ls_command = ["git", "ls-files", "."]
+    ls_result = subprocess.run(
+        ls_command, cwd=ROOT_DIR / path, capture_output=True, check=True, text=True
+    )
+
+    tar_command = [
+        "tar",
+        "-zcf",
+        f"{ROOT_DIR}/dist/{name}.tar.gz",
+        "--transform",
+        f"s,^,{name}/,",
+        "-T",
+        "-",
+    ]
+    subprocess.run(
+        tar_command,
+        cwd=ROOT_DIR / path,
+        input=ls_result.stdout.encode("utf-8"),
+        check=True,
+    )
+
+    print(f"Archive package {path} to dist/{name}.tar.gz")
+
+def generate_signature():
+    for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+        print(f"Generate signature for {i}")
+        subprocess.run(
+            ["gpg", "--yes", "--armor", "--output", f"{i}.asc", "--detach-sig", str(i)],
+            cwd=ROOT_DIR / "dist",
+            check=True,
+        )
+
+    for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+        print(f"Check signature for {i}")
+        subprocess.run(
+            ["gpg", "--verify", f"{i}.asc", str(i)], cwd=ROOT_DIR / "dist", check=True
+        )
+
+
+def generate_checksum():
+    for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+        print(f"Generate checksum for {i}")
+        subprocess.run(
+            ["sha512sum", str(i.relative_to(ROOT_DIR / "dist"))],
+            stdout=open(f"{i}.sha512", "w"),
+            cwd=ROOT_DIR / "dist",
+            check=True,
+        )
+
+    for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+        print(f"Check checksum for {i}")
+        subprocess.run(
+            ["sha512sum", "--check", f"{str(i.relative_to(ROOT_DIR / 'dist'))}.sha512"],
+            cwd=ROOT_DIR / "dist",
+            check=True,
+        )
+
+
+if __name__ == "__main__":
+    for v in PACKAGES:
+        archive_package(v)
+    generate_signature()
+    generate_checksum()
diff --git a/scripts/verify.py b/scripts/verify.py
index 81d6a41ba4..bfca20a520 100755
--- a/scripts/verify.py
+++ b/scripts/verify.py
@@ -47,7 +47,7 @@ def check_java():
 def build_core():
     print("Start building opendal core")
 
-    subprocess.run(["cargo", "build", "--release"], check=True)
+    subprocess.run(["cargo", "build", "--release"], cwd="core", check=True)
 
 
 def build_java_binding():
@@ -62,7 +62,7 @@ def build_java_binding():
             "-Dcargo-build.profile=release",
         ],
         check=True,
-        cwd="bindings/java"
+        cwd="bindings/java",
     )