You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2022/02/16 17:54:30 UTC

[arrow-datafusion] branch master updated: Fix compiling ballista in standalone mode, add build to CI (#1839)

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git


The following commit(s) were added to refs/heads/master by this push:
     new 0a1eee0  Fix compiling ballista  in standalone mode, add build to CI (#1839)
0a1eee0 is described below

commit 0a1eee08de57e1f4a24e74b4ea079226b65fda43
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Wed Feb 16 12:54:22 2022 -0500

    Fix compiling ballista  in standalone mode, add build to CI (#1839)
    
    * Test ballista compiles with --standalone in CI
    
    * Fix compile
    
    * ignore broken test
    !
    
    * add ticket reference
---
 .github/workflows/rust.yml          |  2 ++
 ballista/rust/client/src/context.rs | 31 +++++++++++++++++++++++++------
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 5919f35..1b34d44 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -142,6 +142,8 @@ jobs:
           cd ballista/rust
           # snmalloc requires cmake so build without default features
           cargo test --no-default-features --features sled
+          # Ensure also compiles in standalone mode
+          cargo test --no-default-features --features standalone
         env:
           CARGO_HOME: "/github/home/.cargo"
           CARGO_TARGET_DIR: "/github/home/target"
diff --git a/ballista/rust/client/src/context.rs b/ballista/rust/client/src/context.rs
index 8326afc..69ae971 100644
--- a/ballista/rust/client/src/context.rs
+++ b/ballista/rust/client/src/context.rs
@@ -70,6 +70,8 @@ impl BallistaContextState {
         concurrent_tasks: usize,
     ) -> ballista_core::error::Result<Self> {
         use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient;
+        use ballista_core::serde::protobuf::PhysicalPlanNode;
+        use ballista_core::serde::BallistaCodec;
 
         log::info!("Running in local mode. Scheduler will be run in-proc");
 
@@ -90,7 +92,16 @@ impl BallistaContextState {
             }
         };
 
-        ballista_executor::new_standalone_executor(scheduler, concurrent_tasks).await?;
+        let default_codec: BallistaCodec<LogicalPlanNode, PhysicalPlanNode> =
+            BallistaCodec::default();
+
+        ballista_executor::new_standalone_executor(
+            scheduler,
+            concurrent_tasks,
+            default_codec,
+        )
+        .await?;
+
         Ok(Self {
             config: config.clone(),
             scheduler_host: "localhost".to_string(),
@@ -458,13 +469,17 @@ mod tests {
 
     #[tokio::test]
     #[cfg(feature = "standalone")]
+    #[ignore]
+    // Tracking: https://github.com/apache/arrow-datafusion/issues/1840
     async fn test_task_stuck_when_referenced_task_failed() {
         use super::*;
         use datafusion::arrow::datatypes::Schema;
         use datafusion::arrow::util::pretty;
         use datafusion::datasource::file_format::csv::CsvFormat;
         use datafusion::datasource::file_format::parquet::ParquetFormat;
-        use datafusion::datasource::listing::{ListingOptions, ListingTable};
+        use datafusion::datasource::listing::{
+            ListingOptions, ListingTable, ListingTableConfig,
+        };
 
         use ballista_core::config::{
             BallistaConfigBuilder, BALLISTA_WITH_INFORMATION_SCHEMA,
@@ -502,12 +517,16 @@ mod tests {
                         collect_stat: x.collect_stat,
                         target_partitions: x.target_partitions,
                     };
-                    let error_table = ListingTable::new(
+
+                    let config = ListingTableConfig::new(
                         listing_table.object_store().clone(),
                         listing_table.table_path().to_string(),
-                        Arc::new(Schema::new(vec![])),
-                        error_options,
-                    );
+                    )
+                    .with_schema(Arc::new(Schema::new(vec![])))
+                    .with_listing_options(error_options);
+
+                    let error_table = ListingTable::try_new(config).unwrap();
+
                     // change the table to an error table
                     guard
                         .tables