You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by xu...@apache.org on 2024/07/11 05:52:53 UTC
(hudi-rs) branch main updated: build: add release workflow (#63)
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new cb9f4d8 build: add release workflow (#63)
cb9f4d8 is described below
commit cb9f4d8a64fd6e8c2e06fc2c79a6c6740eeadba4
Author: Shiyan Xu <27...@users.noreply.github.com>
AuthorDate: Thu Jul 11 00:52:47 2024 -0500
build: add release workflow (#63)
- Add `release.yml` workflow in github to publish artifacts to crates.io and pypi.org
- Add various metadata to populate artifact repository main pages
- Fix python artifacts issues wrt macos aarch64
Closes #41 #42
---
.github/workflows/release.yml | 146 ++++++++++++++++++++++++++++++++++++++++++
README.md | 34 +++++-----
crates/core/Cargo.toml | 5 ++
crates/datafusion/Cargo.toml | 7 +-
crates/hudi/Cargo.toml | 12 +++-
crates/tests/Cargo.toml | 5 ++
python/.cargo/config.toml | 1 +
python/Cargo.toml | 12 ++--
python/README.md | 46 -------------
python/pyproject.toml | 7 +-
10 files changed, 199 insertions(+), 76 deletions(-)
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..e7aa911
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,146 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Publish artifacts
+
+on:
+ push:
+ tags:
+ - 'release-[0-9]+.[0-9]+.[0-9]+**'
+
+jobs:
+ validate-release-tag:
+ name: Validate git tag
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - name: compare git tag with cargo metadata
+ run: |
+ CURR_VER=$( grep version Cargo.toml | head -n 1 | awk '{print $3}' | tr -d '"' )
+ if [[ "${GITHUB_REF_NAME}" != "release-${CURR_VER}" ]]; then
+ echo "Pushed tag ${GITHUB_REF_NAME} does not match with the Cargo package version ${CURR_VER}."
+ exit 1
+ fi
+
+ release-crates:
+ name: Release to crates.io
+ needs: validate-release-tag
+ runs-on: ubuntu-latest
+ strategy:
+ max-parallel: 1
+ matrix:
+ # order matters here as later crates depend on previous ones
+ package:
+ - "hudi-core"
+ - "hudi-datafusion"
+ - "hudi"
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions-rs/toolchain@v1
+ with:
+ profile: minimal
+ toolchain: stable
+ override: true
+
+ - name: cargo publish
+ uses: actions-rs/cargo@v1
+ env:
+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+ with:
+ command: publish
+ args: -p ${{ matrix.package }} --all-features
+
+ release-pypi-mac:
+ name: PyPI release on Mac
+ needs: validate-release-tag
+ strategy:
+ fail-fast: false
+ matrix:
+ target: [ x86_64-apple-darwin, aarch64-apple-darwin ]
+ runs-on: macos-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+
+ - name: Publish to pypi (without sdist)
+ uses: PyO3/maturin-action@v1
+ env:
+ MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+ MATURIN_REPOSITORY: pypi
+ with:
+ target: ${{ matrix.target }}
+ command: publish
+ args: --skip-existing -m python/Cargo.toml --no-sdist
+
+ release-pypi-windows:
+ name: PyPI release on Windows
+ needs: validate-release-tag
+ runs-on: windows-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+
+ - name: Publish to pypi (without sdist)
+ uses: PyO3/maturin-action@v1
+ env:
+ MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+ MATURIN_REPOSITORY: pypi
+ with:
+ target: x86_64-pc-windows-msvc
+ command: publish
+ args: --skip-existing -m python/Cargo.toml --no-sdist
+
+ release-pypi-manylinux:
+ name: PyPI release manylinux
+ needs: validate-release-tag
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+
+ - name: Publish manylinux to pypi x86_64 (with sdist)
+ uses: PyO3/maturin-action@v1
+ env:
+ MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+ MATURIN_REPOSITORY: pypi
+ with:
+ target: x86_64-unknown-linux-gnu
+ command: publish
+ args: --skip-existing -m python/Cargo.toml
+ # for openssl build
+ before-script-linux: yum install -y perl-IPC-Cmd
+
+ - name: Publish manylinux to pypi aarch64 (without sdist)
+ uses: PyO3/maturin-action@v1
+ env:
+ MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+ MATURIN_REPOSITORY: pypi
+ with:
+ target: aarch64-unknown-linux-gnu
+ command: publish
+ args: --skip-existing -m python/Cargo.toml --no-sdist
+ before-script-linux: |
+ # We can remove this once we upgrade to 2_28.
+ # https://github.com/briansmith/ring/issues/1728
+ export CFLAGS_aarch64_unknown_linux_gnu="-D__ARM_ARCH=8"
diff --git a/README.md b/README.md
index ad0c251..9658bfd 100644
--- a/README.md
+++ b/README.md
@@ -51,24 +51,11 @@ users and projects.
| **PyPi** | `pip install hudi` |
| **Crates.io** | `cargo add hudi` |
-*Note: not yet available until the first release.*
-
## Example usage
### Python
-Add `python/hudi` as a dependency to your Python application.
-
-For example, use these commands to setup a virtualenv.
-
-```shell
-cd python
-python3 -m venv my_venv
-source my_venv/bin/activate
-pip install -e .
-```
-
-Then query a Hudi table.
+Read a Hudi table into a PyArrow table.
```python
from hudi import HudiTable
@@ -77,30 +64,39 @@ hudi_table = HudiTable("/tmp/trips_table")
records = hudi_table.read_snapshot()
import pyarrow as pa
+import pyarrow.compute as pc
arrow_table = pa.Table.from_batches(records)
result = arrow_table.select(
["rider", "ts", "fare"]).filter(
- pa.compute.field("fare") > 20.0)
+ pc.field("fare") > 20.0)
print(result)
```
### Rust
-Add `hudi-datafusion` as a dependency to your Rust application, and query a Hudi table.
+<details>
+<summary>Add crate `hudi` with `datafusion` feature to your application to query a Hudi table.</summary>
+
+```yaml
+[dependencies]
+hudi = { version = "0" , features = ["datafusion"] }
+tokio = "1"
+datafusion = "39.0.0"
+```
+</details>
```rust
use std::sync::Arc;
use datafusion::error::Result;
use datafusion::prelude::{DataFrame, SessionContext};
-
-use hudi_datafusion::HudiDataSource;
+use hudi::HudiDataSource;
#[tokio::main]
async fn main() -> Result<()> {
let ctx = SessionContext::new();
- let hudi = HudiDataSource::new("/tmp/trips_table");
+ let hudi = HudiDataSource::new("/tmp/trips_table").await?;
ctx.register_table("trips_table", Arc::new(hudi))?;
let df: DataFrame = ctx.sql("SELECT * from trips_table where fare > 20.0").await?;
df.show().await?;
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 151fb2d..ac000ac 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -21,6 +21,11 @@ version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
[dependencies]
# arrow
diff --git a/crates/datafusion/Cargo.toml b/crates/datafusion/Cargo.toml
index de7f2a1..7fa0215 100644
--- a/crates/datafusion/Cargo.toml
+++ b/crates/datafusion/Cargo.toml
@@ -21,9 +21,14 @@ version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
[dependencies]
-hudi-core = { path = "../core" }
+hudi-core = { version = "0.1.0", path = "../core" }
# arrow
arrow-schema = { workspace = true }
diff --git a/crates/hudi/Cargo.toml b/crates/hudi/Cargo.toml
index b30bcb5..df42c13 100644
--- a/crates/hudi/Cargo.toml
+++ b/crates/hudi/Cargo.toml
@@ -21,12 +21,20 @@ version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
[dependencies]
-hudi-core = { path = "../core" }
-hudi-datafusion = { path = "../datafusion", optional = true }
+hudi-core = { version = "0.1.0", path = "../core" }
+hudi-datafusion = { version = "0.1.0", path = "../datafusion", optional = true }
[features]
datafusion = [
"hudi-datafusion",
]
+
+[dev-dependencies]
+tokio = { workspace = true }
diff --git a/crates/tests/Cargo.toml b/crates/tests/Cargo.toml
index 127ec81..66f4f37 100644
--- a/crates/tests/Cargo.toml
+++ b/crates/tests/Cargo.toml
@@ -21,6 +21,11 @@ version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
[dependencies]
# arrow
diff --git a/python/.cargo/config.toml b/python/.cargo/config.toml
index 406230e..65cd4ac 100644
--- a/python/.cargo/config.toml
+++ b/python/.cargo/config.toml
@@ -25,4 +25,5 @@ rustflags = [
rustflags = [
"-C", "link-arg=-undefined",
"-C", "link-arg=dynamic_lookup",
+ "-C", "target-cpu=apple-m1",
]
diff --git a/python/Cargo.toml b/python/Cargo.toml
index 77f16fa..e992c70 100644
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -21,6 +21,11 @@ version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+description = "Native Python binding for Apache Hudi, based on hudi-rs."
+homepage.workspace = true
+repository = "https://github.com/apache/hudi-rs/tree/main/python/"
[lib]
name = "hudi"
@@ -28,8 +33,10 @@ crate-type = ["cdylib"]
doc = false
[dependencies]
+hudi = { path = "../crates/hudi" }
# arrow
arrow = { workspace = true }
+arrow-schema = { workspace = true }
# "stdlib"
anyhow = { workspace = true }
@@ -40,7 +47,4 @@ tokio = { workspace = true }
[dependencies.pyo3]
version = "0.21.2"
-features = ["extension-module", "abi3", "abi3-py38", "anyhow"]
-
-[dependencies.hudi]
-path = "../crates/hudi"
+features = ["extension-module", "abi3", "abi3-py38", "gil-refs", "anyhow"]
diff --git a/python/README.md b/python/README.md
deleted file mode 100644
index 1ff25e2..0000000
--- a/python/README.md
+++ /dev/null
@@ -1,46 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
--->
-
-# hudi-python
-
-Native [Apache Hudi](https://github.com/apache/hudi) Python binding based
-on [hudi-rs](https://github.com/apache/hudi-rs).
-
-## Installation
-
-```shell
-pip install hudi
-```
-
-## Example
-
-```python
-from hudi import HudiTable
-
-hudi_table = HudiTable("/tmp/trips_table")
-records = hudi_table.read_snapshot()
-
-import pyarrow as pa
-
-arrow_table = pa.Table.from_batches(records)
-result = arrow_table.select(
- ["rider", "ts", "fare"]).filter(
- pa.compute.field("fare") > 20.0)
-print(result)
-```
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 13c1c8b..36e44d5 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -21,9 +21,8 @@ build-backend = "maturin"
[project]
name = "hudi"
-description = "Native Hudi Python binding based on hudi-rs"
+description = "Native Python binding for Apache Hudi, based on hudi-rs."
urls = { repository = "https://github.com/apache/hudi-rs/tree/main/python/" }
-readme = "README.md"
requires-python = ">=3.8"
keywords = ["apachehudi", "hudi", "datalake", "arrow"]
license = "Apache License 2.0"
@@ -36,7 +35,8 @@ classifiers = [
"Programming Language :: Python :: 3.12"
]
dependencies = [
- "pyarrow>=8"
+ "pyarrow>=8",
+ "pyarrow-hotfix",
]
optional-dependencies = { devel = [
@@ -47,7 +47,6 @@ dynamic = ["version"]
[tool.maturin]
module-name = "hudi._internal"
-features = ["pyo3/extension-module"]
[tool.mypy]
files = "hudi/*.py"