You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2022/02/16 17:54:30 UTC
[arrow-datafusion] branch master updated: Fix compiling ballista in standalone mode, add build to CI (#1839)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 0a1eee0 Fix compiling ballista in standalone mode, add build to CI (#1839)
0a1eee0 is described below
commit 0a1eee08de57e1f4a24e74b4ea079226b65fda43
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Wed Feb 16 12:54:22 2022 -0500
Fix compiling ballista in standalone mode, add build to CI (#1839)
* Test ballista compiles with --standalone in CI
* Fix compile
* ignore broken test
!
* add ticket reference
---
.github/workflows/rust.yml | 2 ++
ballista/rust/client/src/context.rs | 31 +++++++++++++++++++++++++------
2 files changed, 27 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 5919f35..1b34d44 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -142,6 +142,8 @@ jobs:
cd ballista/rust
# snmalloc requires cmake so build without default features
cargo test --no-default-features --features sled
+ # Ensure also compiles in standalone mode
+ cargo test --no-default-features --features standalone
env:
CARGO_HOME: "/github/home/.cargo"
CARGO_TARGET_DIR: "/github/home/target"
diff --git a/ballista/rust/client/src/context.rs b/ballista/rust/client/src/context.rs
index 8326afc..69ae971 100644
--- a/ballista/rust/client/src/context.rs
+++ b/ballista/rust/client/src/context.rs
@@ -70,6 +70,8 @@ impl BallistaContextState {
concurrent_tasks: usize,
) -> ballista_core::error::Result<Self> {
use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient;
+ use ballista_core::serde::protobuf::PhysicalPlanNode;
+ use ballista_core::serde::BallistaCodec;
log::info!("Running in local mode. Scheduler will be run in-proc");
@@ -90,7 +92,16 @@ impl BallistaContextState {
}
};
- ballista_executor::new_standalone_executor(scheduler, concurrent_tasks).await?;
+ let default_codec: BallistaCodec<LogicalPlanNode, PhysicalPlanNode> =
+ BallistaCodec::default();
+
+ ballista_executor::new_standalone_executor(
+ scheduler,
+ concurrent_tasks,
+ default_codec,
+ )
+ .await?;
+
Ok(Self {
config: config.clone(),
scheduler_host: "localhost".to_string(),
@@ -458,13 +469,17 @@ mod tests {
#[tokio::test]
#[cfg(feature = "standalone")]
+ #[ignore]
+ // Tracking: https://github.com/apache/arrow-datafusion/issues/1840
async fn test_task_stuck_when_referenced_task_failed() {
use super::*;
use datafusion::arrow::datatypes::Schema;
use datafusion::arrow::util::pretty;
use datafusion::datasource::file_format::csv::CsvFormat;
use datafusion::datasource::file_format::parquet::ParquetFormat;
- use datafusion::datasource::listing::{ListingOptions, ListingTable};
+ use datafusion::datasource::listing::{
+ ListingOptions, ListingTable, ListingTableConfig,
+ };
use ballista_core::config::{
BallistaConfigBuilder, BALLISTA_WITH_INFORMATION_SCHEMA,
@@ -502,12 +517,16 @@ mod tests {
collect_stat: x.collect_stat,
target_partitions: x.target_partitions,
};
- let error_table = ListingTable::new(
+
+ let config = ListingTableConfig::new(
listing_table.object_store().clone(),
listing_table.table_path().to_string(),
- Arc::new(Schema::new(vec![])),
- error_options,
- );
+ )
+ .with_schema(Arc::new(Schema::new(vec![])))
+ .with_listing_options(error_options);
+
+ let error_table = ListingTable::try_new(config).unwrap();
+
// change the table to an error table
guard
.tables