You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2021/04/23 12:37:32 UTC
[arrow-datafusion] branch master updated: Combine Cargo workspaces
(#23)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 74cdf6f Combine Cargo workspaces (#23)
74cdf6f is described below
commit 74cdf6f667193a2d26a745632c5a6af79d15f1aa
Author: Andy Grove <an...@users.noreply.github.com>
AuthorDate: Fri Apr 23 06:37:25 2021 -0600
Combine Cargo workspaces (#23)
---
Cargo.toml | 16 ++++++------
ballista/rust/Cargo.toml | 30 ----------------------
ballista/rust/benchmarks/tpch/Cargo.toml | 2 +-
ballista/rust/client/Cargo.toml | 2 +-
ballista/rust/client/src/context.rs | 4 +--
ballista/rust/core/Cargo.toml | 2 +-
ballista/rust/core/src/datasource.rs | 6 ++---
ballista/rust/core/src/error.rs | 1 +
.../rust/core/src/serde/logical_plan/from_proto.rs | 2 ++
.../rust/core/src/serde/logical_plan/to_proto.rs | 7 ++---
.../rust/core/src/serde/scheduler/from_proto.rs | 1 +
ballista/rust/core/src/serde/scheduler/mod.rs | 1 +
ballista/rust/core/src/serde/scheduler/to_proto.rs | 2 ++
ballista/rust/executor/Cargo.toml | 3 +--
ballista/rust/scheduler/Cargo.toml | 2 +-
ballista/rust/scheduler/src/planner.rs | 4 +--
dev/docker/rust.dockerfile | 8 +++---
17 files changed, 34 insertions(+), 59 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index 0a4ef2a..0947bea 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,12 +17,12 @@
[workspace]
members = [
- "datafusion",
- "datafusion-examples",
+ "datafusion",
+ "datafusion-examples",
"benchmarks",
-]
-
-# this package is excluded because it requires different compilation flags, thereby significantly changing
-# how it is compiled within the workspace, causing the whole workspace to be compiled from scratch
-# this way, this is a stand-alone package that compiles independently of the others.
-exclude = ["ballista"]
+ "ballista/rust/benchmarks/tpch",
+ "ballista/rust/client",
+ "ballista/rust/core",
+ "ballista/rust/executor",
+ "ballista/rust/scheduler",
+]
\ No newline at end of file
diff --git a/ballista/rust/Cargo.toml b/ballista/rust/Cargo.toml
deleted file mode 100644
index 5e344e0..0000000
--- a/ballista/rust/Cargo.toml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[workspace]
-
-members = [
- "benchmarks/tpch",
- "client",
- "core",
- "executor",
- "scheduler",
-]
-
-#[profile.release]
-#lto = true
-#codegen-units = 1
diff --git a/ballista/rust/benchmarks/tpch/Cargo.toml b/ballista/rust/benchmarks/tpch/Cargo.toml
index 601943d..9311f23 100644
--- a/ballista/rust/benchmarks/tpch/Cargo.toml
+++ b/ballista/rust/benchmarks/tpch/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "tpch"
-version = "0.4.2-SNAPSHOT"
+version = "0.5.0-SNAPSHOT"
homepage = "https://github.com/apache/arrow"
repository = "https://github.com/apache/arrow"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
diff --git a/ballista/rust/client/Cargo.toml b/ballista/rust/client/Cargo.toml
index fdce1c4..d29f23a 100644
--- a/ballista/rust/client/Cargo.toml
+++ b/ballista/rust/client/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista"
description = "Ballista Distributed Compute"
license = "Apache-2.0"
-version = "0.4.2-SNAPSHOT"
+version = "0.5.0-SNAPSHOT"
homepage = "https://github.com/apache/arrow"
repository = "https://github.com/apache/arrow"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
diff --git a/ballista/rust/client/src/context.rs b/ballista/rust/client/src/context.rs
index 400f6b6..a4cca7a 100644
--- a/ballista/rust/client/src/context.rs
+++ b/ballista/rust/client/src/context.rs
@@ -30,7 +30,7 @@ use ballista_core::serde::protobuf::{
};
use ballista_core::{
client::BallistaClient,
- datasource::DFTableAdapter,
+ datasource::DfTableAdapter,
error::{BallistaError, Result},
memory_stream::MemoryStream,
utils::create_datafusion_context,
@@ -151,7 +151,7 @@ impl BallistaContext {
let execution_plan = ctx.create_physical_plan(&plan)?;
ctx.register_table(
TableReference::Bare { table: name },
- Arc::new(DFTableAdapter::new(plan, execution_plan)),
+ Arc::new(DfTableAdapter::new(plan, execution_plan)),
)?;
}
let df = ctx.sql(sql)?;
diff --git a/ballista/rust/core/Cargo.toml b/ballista/rust/core/Cargo.toml
index 5a77691..0a600ea 100644
--- a/ballista/rust/core/Cargo.toml
+++ b/ballista/rust/core/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-core"
description = "Ballista Distributed Compute"
license = "Apache-2.0"
-version = "0.4.2-SNAPSHOT"
+version = "0.5.0-SNAPSHOT"
homepage = "https://github.com/apache/arrow"
repository = "https://github.com/apache/arrow"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
diff --git a/ballista/rust/core/src/datasource.rs b/ballista/rust/core/src/datasource.rs
index 8ff0df4..5b1540a 100644
--- a/ballista/rust/core/src/datasource.rs
+++ b/ballista/rust/core/src/datasource.rs
@@ -30,20 +30,20 @@ use datafusion::{
/// TableProvider which is effectively a wrapper around a physical plan. We need to be able to
/// register tables so that we can create logical plans from SQL statements that reference these
/// tables.
-pub struct DFTableAdapter {
+pub struct DfTableAdapter {
/// DataFusion logical plan
pub logical_plan: LogicalPlan,
/// DataFusion execution plan
plan: Arc<dyn ExecutionPlan>,
}
-impl DFTableAdapter {
+impl DfTableAdapter {
pub fn new(logical_plan: LogicalPlan, plan: Arc<dyn ExecutionPlan>) -> Self {
Self { logical_plan, plan }
}
}
-impl TableProvider for DFTableAdapter {
+impl TableProvider for DfTableAdapter {
fn as_any(&self) -> &dyn Any {
self
}
diff --git a/ballista/rust/core/src/error.rs b/ballista/rust/core/src/error.rs
index d0155ce..e16920e 100644
--- a/ballista/rust/core/src/error.rs
+++ b/ballista/rust/core/src/error.rs
@@ -49,6 +49,7 @@ pub enum BallistaError {
TokioError(tokio::task::JoinError),
}
+#[allow(clippy::from_over_into)]
impl<T> Into<Result<T>> for BallistaError {
fn into(self) -> Result<T> {
Err(self)
diff --git a/ballista/rust/core/src/serde/logical_plan/from_proto.rs b/ballista/rust/core/src/serde/logical_plan/from_proto.rs
index 9308426..18a85d2 100644
--- a/ballista/rust/core/src/serde/logical_plan/from_proto.rs
+++ b/ballista/rust/core/src/serde/logical_plan/from_proto.rs
@@ -436,6 +436,7 @@ impl TryInto<arrow::datatypes::DataType> for &protobuf::arrow_type::ArrowTypeEnu
}
}
+#[allow(clippy::from_over_into)]
impl Into<arrow::datatypes::DataType> for protobuf::PrimitiveScalarType {
fn into(self) -> arrow::datatypes::DataType {
use arrow::datatypes::DataType;
@@ -1170,6 +1171,7 @@ impl TryFrom<i32> for protobuf::FileType {
}
}
+#[allow(clippy::from_over_into)]
impl Into<datafusion::sql::parser::FileType> for protobuf::FileType {
fn into(self) -> datafusion::sql::parser::FileType {
use datafusion::sql::parser::FileType;
diff --git a/ballista/rust/core/src/serde/logical_plan/to_proto.rs b/ballista/rust/core/src/serde/logical_plan/to_proto.rs
index 222b767..560578d 100644
--- a/ballista/rust/core/src/serde/logical_plan/to_proto.rs
+++ b/ballista/rust/core/src/serde/logical_plan/to_proto.rs
@@ -24,7 +24,7 @@ use std::{
convert::{TryFrom, TryInto},
};
-use crate::datasource::DFTableAdapter;
+use crate::datasource::DfTableAdapter;
use crate::serde::{protobuf, BallistaError};
use arrow::datatypes::{DataType, Schema};
@@ -679,7 +679,7 @@ impl TryInto<protobuf::LogicalPlanNode> for &LogicalPlan {
// unwrap the DFTableAdapter to get to the real TableProvider
let source = if let Some(adapter) =
- source.as_any().downcast_ref::<DFTableAdapter>()
+ source.as_any().downcast_ref::<DfTableAdapter>()
{
match &adapter.logical_plan {
LogicalPlan::TableScan { source, .. } => Ok(source.as_any()),
@@ -1021,7 +1021,7 @@ impl TryInto<protobuf::LogicalExprNode> for &Expr {
let fun: protobuf::ScalarFunction = fun.try_into()?;
let expr: Vec<protobuf::LogicalExprNode> = args
.iter()
- .map(|e| Ok(e.try_into()?))
+ .map(|e| e.try_into())
.collect::<Result<Vec<protobuf::LogicalExprNode>, BallistaError>>()?;
Ok(protobuf::LogicalExprNode {
expr_type: Some(
@@ -1164,6 +1164,7 @@ impl TryInto<protobuf::LogicalExprNode> for &Expr {
}
}
+#[allow(clippy::from_over_into)]
impl Into<protobuf::Schema> for &Schema {
fn into(self) -> protobuf::Schema {
protobuf::Schema {
diff --git a/ballista/rust/core/src/serde/scheduler/from_proto.rs b/ballista/rust/core/src/serde/scheduler/from_proto.rs
index fb1e4f8..4631b2e 100644
--- a/ballista/rust/core/src/serde/scheduler/from_proto.rs
+++ b/ballista/rust/core/src/serde/scheduler/from_proto.rs
@@ -72,6 +72,7 @@ impl TryInto<PartitionId> for protobuf::PartitionId {
}
}
+#[allow(clippy::from_over_into)]
impl Into<PartitionStats> for protobuf::PartitionStats {
fn into(self) -> PartitionStats {
PartitionStats::new(
diff --git a/ballista/rust/core/src/serde/scheduler/mod.rs b/ballista/rust/core/src/serde/scheduler/mod.rs
index 81d8722..bbbd48b 100644
--- a/ballista/rust/core/src/serde/scheduler/mod.rs
+++ b/ballista/rust/core/src/serde/scheduler/mod.rs
@@ -75,6 +75,7 @@ pub struct ExecutorMeta {
pub port: u16,
}
+#[allow(clippy::from_over_into)]
impl Into<protobuf::ExecutorMetadata> for ExecutorMeta {
fn into(self) -> protobuf::ExecutorMetadata {
protobuf::ExecutorMetadata {
diff --git a/ballista/rust/core/src/serde/scheduler/to_proto.rs b/ballista/rust/core/src/serde/scheduler/to_proto.rs
index f581bec..40ca907 100644
--- a/ballista/rust/core/src/serde/scheduler/to_proto.rs
+++ b/ballista/rust/core/src/serde/scheduler/to_proto.rs
@@ -55,6 +55,7 @@ impl TryInto<protobuf::ExecutePartition> for ExecutePartition {
}
}
+#[allow(clippy::from_over_into)]
impl Into<protobuf::PartitionId> for PartitionId {
fn into(self) -> protobuf::PartitionId {
protobuf::PartitionId {
@@ -77,6 +78,7 @@ impl TryInto<protobuf::PartitionLocation> for PartitionLocation {
}
}
+#[allow(clippy::from_over_into)]
impl Into<protobuf::PartitionStats> for PartitionStats {
fn into(self) -> protobuf::PartitionStats {
let none_value = -1_i64;
diff --git a/ballista/rust/executor/Cargo.toml b/ballista/rust/executor/Cargo.toml
index 3df5f6b..15bef69 100644
--- a/ballista/rust/executor/Cargo.toml
+++ b/ballista/rust/executor/Cargo.toml
@@ -19,14 +19,13 @@
name = "ballista-executor"
description = "Ballista Distributed Compute - Executor"
license = "Apache-2.0"
-version = "0.4.2-SNAPSHOT"
+version = "0.5.0-SNAPSHOT"
homepage = "https://github.com/apache/arrow"
repository = "https://github.com/apache/arrow"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
edition = "2018"
[features]
-default = ["snmalloc"]
snmalloc = ["snmalloc-rs"]
[dependencies]
diff --git a/ballista/rust/scheduler/Cargo.toml b/ballista/rust/scheduler/Cargo.toml
index 6c7fc58..342f215 100644
--- a/ballista/rust/scheduler/Cargo.toml
+++ b/ballista/rust/scheduler/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-scheduler"
description = "Ballista Distributed Compute - Scheduler"
license = "Apache-2.0"
-version = "0.4.2-SNAPSHOT"
+version = "0.5.0-SNAPSHOT"
homepage = "https://github.com/apache/arrow"
repository = "https://github.com/apache/arrow"
authors = ["Apache Arrow <de...@arrow.apache.org>"]
diff --git a/ballista/rust/scheduler/src/planner.rs b/ballista/rust/scheduler/src/planner.rs
index e9f668a..e791fa8 100644
--- a/ballista/rust/scheduler/src/planner.rs
+++ b/ballista/rust/scheduler/src/planner.rs
@@ -25,7 +25,7 @@ use std::time::Instant;
use std::{collections::HashMap, future::Future};
use ballista_core::client::BallistaClient;
-use ballista_core::datasource::DFTableAdapter;
+use ballista_core::datasource::DfTableAdapter;
use ballista_core::error::{BallistaError, Result};
use ballista_core::serde::scheduler::ExecutorMeta;
use ballista_core::serde::scheduler::PartitionId;
@@ -138,7 +138,7 @@ impl DistributedPlanner {
stages.append(&mut child_stages);
}
- if let Some(adapter) = execution_plan.as_any().downcast_ref::<DFTableAdapter>() {
+ if let Some(adapter) = execution_plan.as_any().downcast_ref::<DfTableAdapter>() {
// remove Repartition rule because that isn't supported yet
let rules: Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>> = vec![
Arc::new(CoalesceBatches::new()),
diff --git a/dev/docker/rust.dockerfile b/dev/docker/rust.dockerfile
index 19dd487..6505f3c 100644
--- a/dev/docker/rust.dockerfile
+++ b/dev/docker/rust.dockerfile
@@ -59,20 +59,18 @@ ARG RELEASE_FLAG=--release
# force build.rs to run to generate configure_me code.
ENV FORCE_REBUILD='true'
RUN cargo build $RELEASE_FLAG
-RUN cd ballista/rust && \
- cargo build $RELEASE_FLAG
# put the executor on /executor (need to be copied from different places depending on FLAG)
ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-executor /executor; else mv /tmp/ballista/ballista/rust/target/release/ballista-executor /executor; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-executor /executor; else mv /tmp/ballista/target/release/ballista-executor /executor; fi
# put the scheduler on /scheduler (need to be copied from different places depending on FLAG)
ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/ballista/rust/target/release/ballista-scheduler /scheduler; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/target/release/ballista-scheduler /scheduler; fi
# put the tpch on /tpch (need to be copied from different places depending on FLAG)
ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/tpch /tpch; else mv /tmp/ballista/ballista/rust/target/release/tpch /tpch; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi
# Copy the binary into a new container for a smaller docker image
FROM ballistacompute/rust-base:0.4.0-20210213