You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by xu...@apache.org on 2024/07/11 05:52:53 UTC

(hudi-rs) branch main updated: build: add release workflow (#63)

This is an automated email from the ASF dual-hosted git repository.

xushiyan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git


The following commit(s) were added to refs/heads/main by this push:
     new cb9f4d8  build: add release workflow (#63)
cb9f4d8 is described below

commit cb9f4d8a64fd6e8c2e06fc2c79a6c6740eeadba4
Author: Shiyan Xu <27...@users.noreply.github.com>
AuthorDate: Thu Jul 11 00:52:47 2024 -0500

    build: add release workflow (#63)
    
    - Add `release.yml` workflow in github to publish artifacts to crates.io and pypi.org
    - Add various metadata to populate artifact repository main pages
    - Fix python artifacts issues wrt macos aarch64
    
    Closes #41 #42
---
 .github/workflows/release.yml | 146 ++++++++++++++++++++++++++++++++++++++++++
 README.md                     |  34 +++++-----
 crates/core/Cargo.toml        |   5 ++
 crates/datafusion/Cargo.toml  |   7 +-
 crates/hudi/Cargo.toml        |  12 +++-
 crates/tests/Cargo.toml       |   5 ++
 python/.cargo/config.toml     |   1 +
 python/Cargo.toml             |  12 ++--
 python/README.md              |  46 -------------
 python/pyproject.toml         |   7 +-
 10 files changed, 199 insertions(+), 76 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..e7aa911
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,146 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Publish artifacts
+
+on:
+  push:
+    tags:
+      - 'release-[0-9]+.[0-9]+.[0-9]+**'
+
+jobs:
+  validate-release-tag:
+    name: Validate git tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: compare git tag with cargo metadata
+        run: |
+          CURR_VER=$( grep version Cargo.toml | head -n 1 | awk '{print $3}' | tr -d '"' )
+          if [[ "${GITHUB_REF_NAME}" != "release-${CURR_VER}" ]]; then
+            echo "Pushed tag ${GITHUB_REF_NAME} does not match with the Cargo package version ${CURR_VER}."
+            exit 1
+          fi
+
+  release-crates:
+    name: Release to crates.io
+    needs: validate-release-tag
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1
+      matrix:
+        # order matters here as later crates depend on previous ones
+        package:
+          - "hudi-core"
+          - "hudi-datafusion"
+          - "hudi"
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: cargo publish
+        uses: actions-rs/cargo@v1
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        with:
+          command: publish
+          args: -p ${{ matrix.package }} --all-features
+
+  release-pypi-mac:
+    name: PyPI release on Mac
+    needs: validate-release-tag
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [ x86_64-apple-darwin, aarch64-apple-darwin ]
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Publish to pypi (without sdist)
+        uses: PyO3/maturin-action@v1
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+          MATURIN_REPOSITORY: pypi
+        with:
+          target: ${{ matrix.target }}
+          command: publish
+          args: --skip-existing -m python/Cargo.toml --no-sdist
+
+  release-pypi-windows:
+    name: PyPI release on Windows
+    needs: validate-release-tag
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Publish to pypi (without sdist)
+        uses: PyO3/maturin-action@v1
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+          MATURIN_REPOSITORY: pypi
+        with:
+          target: x86_64-pc-windows-msvc
+          command: publish
+          args: --skip-existing -m python/Cargo.toml --no-sdist
+
+  release-pypi-manylinux:
+    name: PyPI release manylinux
+    needs: validate-release-tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Publish manylinux to pypi x86_64 (with sdist)
+        uses: PyO3/maturin-action@v1
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+          MATURIN_REPOSITORY: pypi
+        with:
+          target: x86_64-unknown-linux-gnu
+          command: publish
+          args: --skip-existing -m python/Cargo.toml
+          # for openssl build
+          before-script-linux: yum install -y perl-IPC-Cmd
+
+      - name: Publish manylinux to pypi aarch64 (without sdist)
+        uses: PyO3/maturin-action@v1
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+          MATURIN_REPOSITORY: pypi
+        with:
+          target: aarch64-unknown-linux-gnu
+          command: publish
+          args: --skip-existing -m python/Cargo.toml --no-sdist
+          before-script-linux: |
+            # We can remove this once we upgrade to 2_28.
+            # https://github.com/briansmith/ring/issues/1728
+            export CFLAGS_aarch64_unknown_linux_gnu="-D__ARM_ARCH=8"
diff --git a/README.md b/README.md
index ad0c251..9658bfd 100644
--- a/README.md
+++ b/README.md
@@ -51,24 +51,11 @@ users and projects.
 | **PyPi**      | `pip install hudi`   |
 | **Crates.io** | `cargo add hudi`     |
 
-*Note: not yet available until the first release.*
-
 ## Example usage
 
 ### Python
 
-Add `python/hudi` as a dependency to your Python application.
-
-For example, use these commands to setup a virtualenv.
-
-```shell
-cd python
-python3 -m venv my_venv
-source my_venv/bin/activate
-pip install -e .
-```
-
-Then query a Hudi table.
+Read a Hudi table into a PyArrow table.
 
 ```python
 from hudi import HudiTable
@@ -77,30 +64,39 @@ hudi_table = HudiTable("/tmp/trips_table")
 records = hudi_table.read_snapshot()
 
 import pyarrow as pa
+import pyarrow.compute as pc
 
 arrow_table = pa.Table.from_batches(records)
 result = arrow_table.select(
     ["rider", "ts", "fare"]).filter(
-    pa.compute.field("fare") > 20.0)
+    pc.field("fare") > 20.0)
 print(result)
 ```
 
 ### Rust
 
-Add `hudi-datafusion` as a dependency to your Rust application, and query a Hudi table.
+<details>
+<summary>Add crate `hudi` with `datafusion` feature to your application to query a Hudi table.</summary>
+
+```yaml
+[dependencies]
+hudi = { version = "0" , features = ["datafusion"] }
+tokio = "1"
+datafusion = "39.0.0"
+```
+</details>
 
 ```rust
 use std::sync::Arc;
 
 use datafusion::error::Result;
 use datafusion::prelude::{DataFrame, SessionContext};
-
-use hudi_datafusion::HudiDataSource;
+use hudi::HudiDataSource;
 
 #[tokio::main]
 async fn main() -> Result<()> {
     let ctx = SessionContext::new();
-    let hudi = HudiDataSource::new("/tmp/trips_table");
+    let hudi = HudiDataSource::new("/tmp/trips_table").await?;
     ctx.register_table("trips_table", Arc::new(hudi))?;
     let df: DataFrame = ctx.sql("SELECT * from trips_table where fare > 20.0").await?;
     df.show().await?;
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 151fb2d..ac000ac 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -21,6 +21,11 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 [dependencies]
 # arrow
diff --git a/crates/datafusion/Cargo.toml b/crates/datafusion/Cargo.toml
index de7f2a1..7fa0215 100644
--- a/crates/datafusion/Cargo.toml
+++ b/crates/datafusion/Cargo.toml
@@ -21,9 +21,14 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 [dependencies]
-hudi-core = { path = "../core" }
+hudi-core = { version = "0.1.0", path = "../core" }
 # arrow
 arrow-schema = { workspace = true }
 
diff --git a/crates/hudi/Cargo.toml b/crates/hudi/Cargo.toml
index b30bcb5..df42c13 100644
--- a/crates/hudi/Cargo.toml
+++ b/crates/hudi/Cargo.toml
@@ -21,12 +21,20 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 [dependencies]
-hudi-core = { path = "../core" }
-hudi-datafusion = { path = "../datafusion", optional = true }
+hudi-core = { version = "0.1.0", path = "../core" }
+hudi-datafusion = { version = "0.1.0", path = "../datafusion", optional = true }
 
 [features]
 datafusion = [
     "hudi-datafusion",
 ]
+
+[dev-dependencies]
+tokio = { workspace = true }
diff --git a/crates/tests/Cargo.toml b/crates/tests/Cargo.toml
index 127ec81..66f4f37 100644
--- a/crates/tests/Cargo.toml
+++ b/crates/tests/Cargo.toml
@@ -21,6 +21,11 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 [dependencies]
 # arrow
diff --git a/python/.cargo/config.toml b/python/.cargo/config.toml
index 406230e..65cd4ac 100644
--- a/python/.cargo/config.toml
+++ b/python/.cargo/config.toml
@@ -25,4 +25,5 @@ rustflags = [
 rustflags = [
   "-C", "link-arg=-undefined",
   "-C", "link-arg=dynamic_lookup",
+  "-C", "target-cpu=apple-m1",
 ]
diff --git a/python/Cargo.toml b/python/Cargo.toml
index 77f16fa..e992c70 100644
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -21,6 +21,11 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description = "Native Python binding for Apache Hudi, based on hudi-rs."
+homepage.workspace = true
+repository = "https://github.com/apache/hudi-rs/tree/main/python/"
 
 [lib]
 name = "hudi"
@@ -28,8 +33,10 @@ crate-type = ["cdylib"]
 doc = false
 
 [dependencies]
+hudi = { path = "../crates/hudi" }
 # arrow
 arrow = { workspace = true }
+arrow-schema = { workspace = true }
 
 # "stdlib"
 anyhow = { workspace = true }
@@ -40,7 +47,4 @@ tokio = { workspace = true }
 
 [dependencies.pyo3]
 version = "0.21.2"
-features = ["extension-module", "abi3", "abi3-py38", "anyhow"]
-
-[dependencies.hudi]
-path = "../crates/hudi"
+features = ["extension-module", "abi3", "abi3-py38", "gil-refs", "anyhow"]
diff --git a/python/README.md b/python/README.md
deleted file mode 100644
index 1ff25e2..0000000
--- a/python/README.md
+++ /dev/null
@@ -1,46 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~   http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing,
-  ~ software distributed under the License is distributed on an
-  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  ~ KIND, either express or implied.  See the License for the
-  ~ specific language governing permissions and limitations
-  ~ under the License.
--->
-
-# hudi-python
-
-Native [Apache Hudi](https://github.com/apache/hudi) Python binding based
-on [hudi-rs](https://github.com/apache/hudi-rs).
-
-## Installation
-
-```shell
-pip install hudi
-```
-
-## Example
-
-```python
-from hudi import HudiTable
-
-hudi_table = HudiTable("/tmp/trips_table")
-records = hudi_table.read_snapshot()
-
-import pyarrow as pa
-
-arrow_table = pa.Table.from_batches(records)
-result = arrow_table.select(
-    ["rider", "ts", "fare"]).filter(
-    pa.compute.field("fare") > 20.0)
-print(result)
-```
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 13c1c8b..36e44d5 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -21,9 +21,8 @@ build-backend = "maturin"
 
 [project]
 name = "hudi"
-description = "Native Hudi Python binding based on hudi-rs"
+description = "Native Python binding for Apache Hudi, based on hudi-rs."
 urls = { repository = "https://github.com/apache/hudi-rs/tree/main/python/" }
-readme = "README.md"
 requires-python = ">=3.8"
 keywords = ["apachehudi", "hudi", "datalake", "arrow"]
 license = "Apache License 2.0"
@@ -36,7 +35,8 @@ classifiers = [
     "Programming Language :: Python :: 3.12"
 ]
 dependencies = [
-    "pyarrow>=8"
+    "pyarrow>=8",
+    "pyarrow-hotfix",
 ]
 
 optional-dependencies = { devel = [
@@ -47,7 +47,6 @@ dynamic = ["version"]
 
 [tool.maturin]
 module-name = "hudi._internal"
-features = ["pyo3/extension-module"]
 
 [tool.mypy]
 files = "hudi/*.py"