You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2021/08/15 11:35:53 UTC
[arrow-datafusion] branch master updated: prepare code base for
5.1.0 release (#885)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 235e36e prepare code base for 5.1.0 release (#885)
235e36e is described below
commit 235e36ec06a754c2e3322a8ee2590497eba70b12
Author: QP Hou <qp...@scribd.com>
AuthorDate: Sun Aug 15 04:35:50 2021 -0700
prepare code base for 5.1.0 release (#885)
---
ballista-examples/Cargo.toml | 2 +-
ballista/rust/client/Cargo.toml | 10 ++---
ballista/rust/core/Cargo.toml | 4 +-
ballista/rust/executor/Cargo.toml | 6 +--
ballista/rust/scheduler/Cargo.toml | 8 ++--
datafusion-cli/Cargo.toml | 2 +-
datafusion/Cargo.toml | 2 +-
dev/update_ballista_versions.py | 25 ++++++++----
dev/update_datafusion_versions.py | 84 ++++++++++++++++++++++++++++++++++++++
9 files changed, 117 insertions(+), 26 deletions(-)
diff --git a/ballista-examples/Cargo.toml b/ballista-examples/Cargo.toml
index c272225..50b5eec 100644
--- a/ballista-examples/Cargo.toml
+++ b/ballista-examples/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "ballista-examples"
description = "Ballista usage examples"
-version = "0.5.0"
+version = "0.6.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
diff --git a/ballista/rust/client/Cargo.toml b/ballista/rust/client/Cargo.toml
index 7b22bae..d381085 100644
--- a/ballista/rust/client/Cargo.toml
+++ b/ballista/rust/client/Cargo.toml
@@ -19,21 +19,21 @@
name = "ballista"
description = "Ballista Distributed Compute"
license = "Apache-2.0"
-version = "0.5.0"
+version = "0.6.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
edition = "2018"
[dependencies]
-ballista-core = { path = "../core", version = "0.5.0" }
-ballista-executor = { path = "../executor", version = "0.5.0", optional = true }
-ballista-scheduler = { path = "../scheduler", version = "0.5.0", optional = true }
+ballista-core = { path = "../core", version = "0.6.0" }
+ballista-executor = { path = "../executor", version = "0.6.0", optional = true }
+ballista-scheduler = { path = "../scheduler", version = "0.6.0", optional = true }
futures = "0.3"
log = "0.4"
tokio = "1.0"
-datafusion = { path = "../../../datafusion", version = "5.0.0" }
+datafusion = { path = "../../../datafusion", version = "5.1.0" }
[features]
default = []
diff --git a/ballista/rust/core/Cargo.toml b/ballista/rust/core/Cargo.toml
index b2fa50c..99f38b7 100644
--- a/ballista/rust/core/Cargo.toml
+++ b/ballista/rust/core/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-core"
description = "Ballista Distributed Compute"
license = "Apache-2.0"
-version = "0.5.0"
+version = "0.6.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
@@ -44,7 +44,7 @@ uuid = { version = "0.8", features = ["v4"] }
arrow-flight = { version = "5.0" }
-datafusion = { path = "../../../datafusion", version = "5.0.0" }
+datafusion = { path = "../../../datafusion", version = "5.1.0" }
[dev-dependencies]
tempfile = "3"
diff --git a/ballista/rust/executor/Cargo.toml b/ballista/rust/executor/Cargo.toml
index 8600409..9e14a51 100644
--- a/ballista/rust/executor/Cargo.toml
+++ b/ballista/rust/executor/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-executor"
description = "Ballista Distributed Compute - Executor"
license = "Apache-2.0"
-version = "0.5.0"
+version = "0.6.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
@@ -33,9 +33,9 @@ arrow = { version = "5.0" }
arrow-flight = { version = "5.0" }
anyhow = "1"
async-trait = "0.1.36"
-ballista-core = { path = "../core", version = "0.5.0" }
+ballista-core = { path = "../core", version = "0.6.0" }
configure_me = "0.4.0"
-datafusion = { path = "../../../datafusion", version = "5.0.0" }
+datafusion = { path = "../../../datafusion", version = "5.1.0" }
env_logger = "0.8"
futures = "0.3"
log = "0.4"
diff --git a/ballista/rust/scheduler/Cargo.toml b/ballista/rust/scheduler/Cargo.toml
index c772ba9..b47b016 100644
--- a/ballista/rust/scheduler/Cargo.toml
+++ b/ballista/rust/scheduler/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-scheduler"
description = "Ballista Distributed Compute - Scheduler"
license = "Apache-2.0"
-version = "0.5.0"
+version = "0.6.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
@@ -32,10 +32,10 @@ sled = ["sled_package", "tokio-stream"]
[dependencies]
anyhow = "1"
-ballista-core = { path = "../core", version = "0.5.0" }
+ballista-core = { path = "../core", version = "0.6.0" }
clap = "2"
configure_me = "0.4.0"
-datafusion = { path = "../../../datafusion", version = "5.0.0" }
+datafusion = { path = "../../../datafusion", version = "5.1.0" }
env_logger = "0.8"
etcd-client = { version = "0.6", optional = true }
futures = "0.3"
@@ -55,7 +55,7 @@ tower = { version = "0.4" }
warp = "0.3"
[dev-dependencies]
-ballista-core = { path = "../core", version = "0.5.0" }
+ballista-core = { path = "../core", version = "0.6.0" }
uuid = { version = "0.8", features = ["v4"] }
[build-dependencies]
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index fda9271..d2f3045 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -30,5 +30,5 @@ repository = "https://github.com/apache/arrow-datafusion"
clap = "2.33"
rustyline = "8.0"
tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] }
-datafusion = { path = "../datafusion" }
+datafusion = { path = "../datafusion", version = "5.1.0" }
arrow = { version = "5.0" }
diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml
index 286be8a..db950c4 100644
--- a/datafusion/Cargo.toml
+++ b/datafusion/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "datafusion"
description = "DataFusion is an in-memory query engine that uses Apache Arrow as the memory model"
-version = "5.0.0"
+version = "5.1.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
readme = "../README.md"
diff --git a/dev/update_ballista_versions.py b/dev/update_ballista_versions.py
index 4fa2fc7..3a023a7 100755
--- a/dev/update_ballista_versions.py
+++ b/dev/update_ballista_versions.py
@@ -28,7 +28,7 @@ from pathlib import Path
import tomlkit
-def update_cargo_toml(cargo_toml: str, new_version: str, datafusion_version: str):
+def update_cargo_toml(cargo_toml: str, new_version: str):
print(f'updating {cargo_toml}')
with open(cargo_toml) as f:
data = f.read()
@@ -36,9 +36,19 @@ def update_cargo_toml(cargo_toml: str, new_version: str, datafusion_version: str
doc = tomlkit.parse(data)
doc.get('package')['version'] = new_version
- df_dep = doc.get('dependencies').get('datafusion')
- if df_dep is not None:
- df_dep['version'] = datafusion_version
+ # ballista crates also depend on each other
+ ballista_deps = (
+ 'ballista-core',
+ 'ballista-executor',
+ 'ballista-scheduler',
+ )
+ for ballista_dep in ballista_deps:
+ dep = doc.get('dependencies', {}).get(ballista_dep)
+ if dep is not None:
+ dep['version'] = new_version
+ dep = doc.get('dev-dependencies', {}).get(ballista_dep)
+ if dep is not None:
+ dep['version'] = new_version
with open(cargo_toml, 'w') as f:
f.write(tomlkit.dumps(doc))
@@ -47,7 +57,6 @@ def update_cargo_toml(cargo_toml: str, new_version: str, datafusion_version: str
def main():
parser = argparse.ArgumentParser(description='Update ballista crate versions.')
parser.add_argument('new_version', type=str, help='new ballista version')
- parser.add_argument('datafusion_version', type=str, help='new datafusion version')
args = parser.parse_args()
repo_root = Path(__file__).parent.parent.absolute()
@@ -62,13 +71,11 @@ def main():
]
])
new_version = args.new_version
- datafusion_version = args.datafusion_version
- print(f'Updating ballista versions in {repo_root} to {new_version} '
- f'and datafusion dep version to {datafusion_version}')
+ print(f'Updating ballista versions in {repo_root} to {new_version}')
for cargo_toml in ballista_crates:
- update_cargo_toml(cargo_toml, new_version, datafusion_version)
+ update_cargo_toml(cargo_toml, new_version)
if __name__ == "__main__":
diff --git a/dev/update_datafusion_versions.py b/dev/update_datafusion_versions.py
new file mode 100755
index 0000000..d312f21
--- /dev/null
+++ b/dev/update_datafusion_versions.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Script that updates verions for datafusion crates, locally
+#
+# dependencies:
+# pip install tomlkit
+
+import os
+import argparse
+from pathlib import Path
+import tomlkit
+
+
+def update_datafusion_version(cargo_toml: str, new_version: str):
+ print(f'updating {cargo_toml}')
+ with open(cargo_toml) as f:
+ data = f.read()
+
+ doc = tomlkit.parse(data)
+ doc.get('package')['version'] = new_version
+
+ with open(cargo_toml, 'w') as f:
+ f.write(tomlkit.dumps(doc))
+
+
+def update_downstream_versions(cargo_toml: str, new_version: str):
+ with open(cargo_toml) as f:
+ data = f.read()
+
+ doc = tomlkit.parse(data)
+
+ df_dep = doc.get('dependencies', {}).get('datafusion')
+ # skip crates that pin datafusion using git hash
+ if df_dep is not None and df_dep.get('version') is not None:
+ print(f'updating datafusion dependency in {cargo_toml}')
+ df_dep['version'] = new_version
+
+ df_dep = doc.get('dev-dependencies', {}).get('datafusion')
+ if df_dep is not None and df_dep.get('version') is not None:
+ print(f'updating datafusion dev-dependency in {cargo_toml}')
+ df_dep['version'] = new_version
+
+ with open(cargo_toml, 'w') as f:
+ f.write(tomlkit.dumps(doc))
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description=(
+ 'Update datafusion crate version and corresponding version pins '
+ 'in downstream crates.'
+ ))
+ parser.add_argument('new_version', type=str, help='new datafusion version')
+ args = parser.parse_args()
+
+ new_version = args.new_version
+ repo_root = Path(__file__).parent.parent.absolute()
+
+ print(f'Updating datafusion versions in {repo_root} to {new_version}')
+
+ update_datafusion_version("datafusion/Cargo.toml", new_version)
+ for cargo_toml in repo_root.rglob('Cargo.toml'):
+ update_downstream_versions(cargo_toml, new_version)
+
+
+if __name__ == "__main__":
+ main()