(arrow-datafusion) branch main updated: make format prefix optional for format options in COPY (#9723)

[al...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new 4913a0025b make format prefix optional for format options in COPY (#9723)
4913a0025b is described below

commit 4913a0025b63211d4848693e972540c781c1cf98
Author: Kunal Kundu <ku...@gmail.com>
AuthorDate: Fri Mar 22 23:50:21 2024 +0530

    make format prefix optional for format options in COPY (#9723)
    
    * make format prefix optional for format options in COPY
    
    * fix clippy lint error
    
    * Add negative test case for unknown option
    
    * Improve test comments
    
    ---------
    
    Co-authored-by: Andrew Lamb <an...@nerdnetworks.org>
---
 datafusion/sql/src/statement.rs             | 11 +++++++-
 datafusion/sqllogictest/test_files/copy.slt | 39 +++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
index e50aceb757..4cca4c114a 100644
--- a/datafusion/sql/src/statement.rs
+++ b/datafusion/sql/src/statement.rs
@@ -850,7 +850,16 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                     return plan_err!("Unsupported Value in COPY statement {}", value);
                 }
             };
-            options.insert(key.to_lowercase(), value_string.to_lowercase());
+            if !(&key.contains('.')) {
+                // If config does not belong to any namespace, assume it is
+                // a format option and apply the format prefix for backwards
+                // compatibility.
+
+                let renamed_key = format!("format.{}", key);
+                options.insert(renamed_key.to_lowercase(), value_string.to_lowercase());
+            } else {
+                options.insert(key.to_lowercase(), value_string.to_lowercase());
+            }
         }
 
         let file_type = if let Some(file_type) = statement.stored_as {
diff --git a/datafusion/sqllogictest/test_files/copy.slt b/datafusion/sqllogictest/test_files/copy.slt
index 7884bece1f..75f1ccb07a 100644
--- a/datafusion/sqllogictest/test_files/copy.slt
+++ b/datafusion/sqllogictest/test_files/copy.slt
@@ -474,6 +474,45 @@ select * from validate_arrow;
 1 Foo
 2 Bar
 
+# Format Options Support without the 'format.' prefix
+
+# Copy with format options for Parquet without the 'format.' prefix
+query IT
+COPY source_table TO 'test_files/scratch/copy/format_table.parquet'
+OPTIONS (
+    compression snappy,
+    'compression::col1' 'zstd(5)'
+);
+----
+2
+
+# Copy with format options for JSON without the 'format.' prefix
+query IT
+COPY source_table  to 'test_files/scratch/copy/format_table'
+STORED AS JSON OPTIONS (compression gzip);
+----
+2
+
+# Copy with format options for CSV without the 'format.' prefix
+query IT
+COPY source_table to 'test_files/scratch/copy/format_table.csv'
+OPTIONS (
+    has_header false,
+    compression xz,
+    datetime_format '%FT%H:%M:%S.%9f',
+    delimiter ';',
+    null_value 'NULLVAL'
+);
+----
+2
+
+# Copy with unknown format options without the 'format.' prefix to ensure error is sensible
+query error DataFusion error: Invalid or Unsupported Configuration: Config value "unknown_option" not found on CsvOptions
+COPY source_table to 'test_files/scratch/copy/format_table2.csv'
+OPTIONS (
+    unknown_option  false,
+);
+
 
 # Error cases: