You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2023/06/27 09:45:00 UTC
[arrow-datafusion] branch main updated (9dd3c2b6e9 -> 70d633a71d)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
omit 9dd3c2b6e9 chore(deps): update indexmap requirement from 1.9.2 to 2.0.0 (#6766)
omit 7d3cae0699 feat: add `-c option` to pass the SQL query directly as an argument on datafusion-cli (#6765)
omit f24a724543 Order Preserving RepartitionExec Implementation (#6742)
omit 1522e7a58e Re-export modules individually to fix rustdocs (#6757)
omit 8c7678a0b1 add UT to verify 'issues/6606' (#6762)
omit 2d69ddb67a Docs: Improve documentation for `struct` function` (#6754)
omit 878fec124e Move PartitionStream to physical_plan (#6756)
omit d9d9328cc5 minor: fix typo (#6761)
omit 80d1750fda Docs: Update SQL status page (#6736)
omit 631e643641 Update sqlparser-rs to version `0.35.0` (#6753)
omit e60e9d2eda Add support for order-sensitive aggregation for multipartitions (#6734)
omit 80d47cce60 Move some code before tests module (#6749)
omit 01eb72af4c chore(deps): update itertools requirement from 0.10 to 0.11 (#6752)
omit 8be5a8ced8 Aggregate ordering substrait support (#6745)
omit b1b8c9caa2 Support User Defined Window Functions (#6703)
omit eb290a0bb9 Update documentation for creating User Defined Aggregates (AggregateUDF) (#6729)
omit 89669907c3 Minor: remove dead code with decimal datatype from in_list (#6737)
omit b275e1df43 Minor: remove some unwrap (#6738)
omit 390841172a Fix up some DataFusionError::Internal errors with correct type (#6721)
omit 37154286e9 refactor: extract merge_projection common function. (#6735)
omit 1b4153f1e2 `derive(Debug)` for `Expr` (#6708)
omit c42f5efc99 Return null for date_trunc(null) instead of panic (#6723)
omit 98669b000e Hash Join Vectorized collision checking (#6724)
omit 3c304e0519 Move `PartitionEvaluator` and window_state structures to `datafusion_expr` crate (#6690)
omit e0330d6c95 Update `arrow`, `arrow-flight` and `parquet` to `42.0.0` (#6702)
omit f0e02abb67 Display all partitions and files in EXPLAIN VERBOSE (#6711)
omit d32f9a7425 Update copyight date on website (#6727)
omit da9d2616d3 MINOR: Add maintains input order flag to CoalesceBatches (#6730)
omit 748f4bed8a Support Expr::InList to Substrait::RexType (#6604)
omit 39666eb847 add Eq and PartialEq to ListingTableUrl (#6725)
omit 0ef9478e66 Minor: Improve UX for setting ExecutionProps::query_execution_start_time (#6719)
omit d29ab93150 fix: parser for negative intervals (#6698)
omit cae8cc2764 Move alias generator to per-query execution props (#6706)
omit faac34c5a2 chore(deps): update strum/strum_macros requirement from 0.24 to 0.25 (#6717)
omit feb4b8fbe1 refactor: alias() should skip add alias for `Expr::Sort` (#6707)
omit 26c90c211e Improve performance/memory usage of HashJoin datastructure (5-15% improvement on selected TPC-H queries) (#6679)
omit 873d4178ab Port test in subqueries.rs from rust to sqllogictest (#6675)
omit a3035b510f fix: correct the error type (#6712)
omit 867ce6ee49 minor: fix documentation typo (#6704)
omit ebd0f7e83b refactor: fix clippy allow too many arguments (#6705)
omit 9b419b19a6 Move `dataframe` and `dataframe_functon` into `core_integration` test binary (#6697)
omit 8da5f26a43 Replace supports_bounded_execution with supports_retract_batch (#6695)
omit 4a2c28c173 removed self.all_value.len() from reserve (#6689)
omit 31c96a77ae Minor: consolidate datafusion_substrait tests into `substrait_integration` (#6686)
omit 0d82620bb4 Minor: combine fuzz tests into a single binary to save builder space and build time (#6684)
omit 3ce75b3fa9 Rename integration tests to match crate they are defined in (#6687)
omit b7cd7bae91 Move PhysicalPlanner to `physical_planer` module (#6570)
omit af13ac788c Combine `statistics`, `filter_pushdown` and custom_sources provider tests together into a single test binary (#6683)
omit aa82de105a Minor: consolidate repartition test into sql_integration (#6685)
omit c2b2606207 Minor: Update user guide (#6692)
omit 84e49771b7 Allow `AggregateUDF` to define retractable batch , implement sliding window functions (#6671)
omit 7354c56e2c Disable incremental compilation (#6688)
omit 9dfaf4249e Docs: Update roadmap to point at EPIC's, clarify project goals (#6639)
omit 638aba6335 minor: `with_new_inputs` replace `from_plan` (#6680)
omit 6caa804050 Minor: Add more doc strings to WindowExpr (#6663)
omit 6872535358 feat: new concatenation operator for working with arrays (#6615)
omit 36123ee018 fix: from_plan shouldn't use original schema (#6595)
omit c4928b035d fix: correct wrong test test_count_wildcard_on_where_scalar_subquery (#6667)
omit 8eb51089dd Minor: Add tests for User Defined Aggregate functions (#6669)
omit c3d5d77e44 minor: use sql to setup test data for aggregate.slt rather than rust (#6664)
omit e9fae9889d Support internal cast for BuiltinScalarFunction::MakeArray (#6607)
omit 4184a7f72c Combine evaluate_stateful and evaluate_inside_range (#6665)
omit 6194d588d5 refactor: unify replace count(*) analyzer by removing it in sql crate (#6660)
omit b586d4e179 Minor: add test cases for coercion bitwise shifts (#6651)
omit ee80d064ed refactor: unify generic expr rewrite functions into the `datafusion_expr::expr_rewriter` (#6644)
omit 40c1b9b76e Fix round function doc (#6661)
omit 6558a8216a fix: port unstable subquery to sqllogicaltest (#6659)
omit fd444b4f68 Support wider range of Subquery, handle the Count bug (#6457)
omit 2ba067c8d2 minor: use sql to setup test data for joins.slt rather than rust (#6656)
omit 071a2a6dfd Minor: Move get_equal_orderings into `BuiltInWindowFunctionExpr`, remove `BuiltInWindowFunctionExpr::as_any` (#6619)
omit a56ae74c0d Simplify and encapsulate window function state management (#6621)
omit d602de2ebc Minor: Add test for date_trunc schema on scalars (#6655)
omit 3fc009b41e Port tests in joins.rs to sqllogictes (#6642)
omit 065dba72e9 Add hash support for PhysicalExpr and PhysicalSortExpr (#6625)
omit db70251854 Fix Clippy module inception (unwrap `datasource::datasource` and `catalog::catalog` (#6640)
omit 76c5bb1e5c Insert supports specifying column names in any order (#6628)
omit 9846da6f1f Return correct scalar types for date_trunc (#6638)
omit 6df158eb4b Fix date_trunc signature (#6632)
omit 2d8a42ecf3 refactor: encapsulate physical optimizer rules into a struct (#6645)
omit c96949ea8c Prioritize UDF over scalar built-in function in case of function name… (#6601)
omit d584d55a9d Minor: Move `include_rank` into `BuiltInWindowFunctionExpr` (#6620)
omit b8f90fe936 Improve main api doc page, move `avro_to_arrow` to `datasource` (#6564)
omit 2467226aca chore(deps): update sqllogictest requirement from 0.13.2 to 0.14.0 (#6646)
omit 373e291fab Minor: cleanup the unnecessary CREATE TABLE aggregate_test_100 statement at aggregate.slt
omit d3ef6c88d8 feat: make_array support empty arguments (#6593)
omit 708044ce92 Unify formatting of both groups and files up to 5 elements (#6637)
omit 945191fa6c fix: median with even number of decimal128 not working (#6634)
omit 93b1a57f9a fix: fix more panics in `ListingTable` (#6636)
omit 365336259e Refactor joins test to sqllogic (#6525)
omit a2d3c3c32c Add datafusion-cli tests to the CI Job (#6600)
omit d024f37944 Move functionality into `BuildInScalarFunction` (#6612)
omit aebe71eace Minor: Add debug logging for schema mismatch errors (#6626)
omit e6265c1e3f Substrait: Fix incorrect join key fields (indices) when same table is being used more than once (#6135)
omit 57bc5b0299 style: make clippy happy and remove redundant prefix (#6624)
omit c4a036dabc Impl `Literal` trait for `NonZero*` types (#6627)
omit c166a58420 fix: correct test timestamp_add_interval_months (#6622)
omit f0bf01691f Minor: Move `PlanType`, `StringifiedPlan` and `ToStringifiedPlan` `datafusion_common` (#6571)
omit 1af846bd8d Minor: use upstream `dialect_from_str` (#6616)
omit 005ecdc485 fix: remove type coercion of case expression in Expr::Schema (#6614)
omit d9e91d187c move functions.rs to sqllogictests (#6608)
omit eacbdd9515 Revert "feat: Implement the bitwise_not in NotExpr (#5902)" (#6599)
omit 43423be8e5 [Minor] Cleanup tpch benchmark (#6609)
omit 3d36dfe210 Remove `from_slice` and use upstream (#6587)
omit 5032060c78 Minor: Add additional docstrings to Window function implementations (#6592)
omit 95c4681baf feat: analyze/optimize plan in `CREATE TABLE AS SELECT` (#6610)
omit 1a1efcf726 fix: avoid panic in `list_files_for_scan` (#6605)
omit 34fa0b940b Make the struct function return the correct data type. (#6594)
omit dbd92e60c3 feat: `DISTINCT` bitwise and boolean aggregate functions (#6581)
omit d689daf83f Minor: Remove some `use crate::` uses in physical_plan (#6573)
omit 4973c7c238 make page filter public (#6523)
omit 9c13a771f9 feat: type coercion support date - date (#6578)
omit 1d3860dc81 refactor: bitwise kernel right and left shifts (#6585)
omit 919ac96d89 chore(deps): update substrait requirement from 0.10.0 to 0.11.0 (#6579)
omit cbfb7c8d58 Move JoinType to datafusion_common (#6572)
omit 8f7f76d6e3 Avoid per-batch field lookups in SchemaMapping (#6563)
omit 25b6556c64 Add tests for object_store builders of datafusion-cli (#6576)
omit afa64e0537 minor: remove useless mut and borrow() (#6580)
omit ce76f1b6c5 Minor: remove tokio_stream dependency (#6565)
omit 12b88ea9b3 refactor: remove type_coercion in PhysicalExpr. (#6575)
omit 786f222e23 Move `physical_plan::file_format` to `datasource::plan` (#6516)
omit 39ee59aeb5 Fix panic propagation in `CoalescePartitions`, consolidates panic propagation into `RecordBatchReceiverStream` (#6507)
omit 9cb6161dde refactor: use bitwise and boolean compute functions (#6568)
omit 2eab6d7d3c [Minor] Update hashbrown to 0.14 (#6562)
omit 36292f63e3 Add support for appending data to external tables - CSV (#6526)
omit 44b83a10a6 feat: New functions and operations for working with arrays (#6384)
omit 91e75d7e63 Update to object_store 0.6 and arrow 41 (#6374)
This update removed existing revisions from the reference, leaving the
reference pointing at a previous point in the repository history.
* -- * -- N refs/heads/main (70d633a71d)
\
O -- O -- O (9dd3c2b6e9)
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/actions/setup-builder/action.yaml | 5 -
.github/workflows/rust.yml | 38 +-
Cargo.toml | 13 +-
benchmarks/src/bin/parquet.rs | 2 +-
benchmarks/src/bin/tpch.rs | 4 +-
benchmarks/src/tpch.rs | 299 ++-
ci/scripts/rust_clippy.sh | 2 -
ci/scripts/rust_docs.sh | 2 -
ci/scripts/rust_fmt.sh | 2 -
datafusion-cli/Cargo.lock | 665 ++---
datafusion-cli/Cargo.toml | 12 +-
datafusion-cli/src/catalog.rs | 2 +-
datafusion-cli/src/exec.rs | 68 +-
datafusion-cli/src/main.rs | 30 +-
datafusion-cli/src/object_storage.rs | 162 +-
datafusion-cli/src/print_format.rs | 13 +-
datafusion-cli/tests/cli_integration.rs | 52 -
datafusion-cli/tests/data/sql.txt | 1 -
datafusion-examples/Cargo.toml | 3 +-
datafusion-examples/README.md | 1 -
datafusion-examples/examples/catalog.rs | 2 +-
datafusion-examples/examples/csv_opener.rs | 9 +-
.../examples/dataframe_in_memory.rs | 5 +-
datafusion-examples/examples/flight_sql_server.rs | 93 +-
datafusion-examples/examples/json_opener.rs | 9 +-
.../examples/parquet_sql_multiple_files.rs | 2 +-
datafusion-examples/examples/rewrite_expr.rs | 6 +-
datafusion-examples/examples/simple_udaf.rs | 5 +-
datafusion-examples/examples/simple_udf.rs | 5 +-
datafusion-examples/examples/simple_udwf.rs | 211 --
datafusion/common/Cargo.toml | 6 +-
datafusion/common/src/cast.rs | 7 +-
datafusion/common/src/config.rs | 2 +-
datafusion/common/src/dfschema.rs | 8 +-
datafusion/common/src/display.rs | 110 -
datafusion/common/src/error.rs | 6 +-
datafusion/common/src/from_slice.rs | 116 +
datafusion/common/src/join_type.rs | 98 -
datafusion/common/src/lib.rs | 5 +-
datafusion/common/src/pyarrow.rs | 12 +-
datafusion/common/src/scalar.rs | 25 +-
datafusion/common/src/utils.rs | 44 +-
datafusion/core/Cargo.toml | 13 +-
datafusion/core/benches/data_utils/mod.rs | 3 +-
datafusion/core/benches/filter_query_sql.rs | 5 +-
datafusion/core/benches/math_query_sql.rs | 5 +-
.../avro_to_arrow/arrow_array_reader.rs | 2 +-
.../core/src/{datasource => }/avro_to_arrow/mod.rs | 4 +-
.../src/{datasource => }/avro_to_arrow/reader.rs | 14 +-
.../src/{datasource => }/avro_to_arrow/schema.rs | 0
datafusion/core/src/catalog/{mod.rs => catalog.rs} | 9 +-
datafusion/core/src/catalog/information_schema.rs | 9 +-
datafusion/core/src/catalog/listing_schema.rs | 2 +-
datafusion/core/src/catalog/mod.rs | 268 +-
datafusion/core/src/catalog/schema.rs | 2 +-
datafusion/core/src/dataframe.rs | 19 +-
.../src/datasource/{provider.rs => datasource.rs} | 0
.../core/src/datasource/file_format/arrow.rs | 2 +-
datafusion/core/src/datasource/file_format/avro.rs | 4 +-
datafusion/core/src/datasource/file_format/csv.rs | 337 +--
.../core/src/datasource/file_format/file_type.rs | 30 +-
datafusion/core/src/datasource/file_format/json.rs | 2 +-
datafusion/core/src/datasource/file_format/mod.rs | 244 +-
.../core/src/datasource/file_format/options.rs | 4 +-
.../core/src/datasource/file_format/parquet.rs | 23 +-
datafusion/core/src/datasource/listing/mod.rs | 7 +-
datafusion/core/src/datasource/listing/table.rs | 290 +--
datafusion/core/src/datasource/listing/url.rs | 2 +-
.../core/src/datasource/listing_table_factory.rs | 6 +-
datafusion/core/src/datasource/memory.rs | 108 +-
datafusion/core/src/datasource/mod.rs | 14 +-
datafusion/core/src/datasource/streaming.rs | 31 +-
datafusion/core/src/error.rs | 2 +-
datafusion/core/src/execution/context.rs | 172 +-
datafusion/core/src/lib.rs | 78 +-
.../combine_partial_final_agg.rs | 4 +-
.../src/physical_optimizer/dist_enforcement.rs | 175 +-
.../core/src/physical_optimizer/join_selection.rs | 6 +-
datafusion/core/src/physical_optimizer/mod.rs | 9 +-
.../core/src/physical_optimizer/optimizer.rs | 83 -
datafusion/core/src/physical_optimizer/pruning.rs | 3 +-
.../core/src/physical_optimizer/repartition.rs | 8 +-
.../src/physical_optimizer/sort_enforcement.rs | 57 +-
.../core/src/physical_plan/aggregates/mod.rs | 166 +-
datafusion/core/src/physical_plan/analyze.rs | 50 +-
.../core/src/physical_plan/coalesce_batches.rs | 8 +-
.../core/src/physical_plan/coalesce_partitions.rs | 79 +-
datafusion/core/src/physical_plan/common.rs | 225 +-
datafusion/core/src/physical_plan/display.rs | 55 +-
datafusion/core/src/physical_plan/empty.rs | 4 +-
datafusion/core/src/physical_plan/explain.rs | 12 +-
.../file_format}/arrow_file.rs | 13 +-
.../file_format}/avro.rs | 23 +-
.../file_format}/chunked_store.rs | 6 +-
.../file_format}/csv.rs | 28 +-
.../file_format}/file_stream.rs | 54 +-
.../file_format}/json.rs | 19 +-
.../file_format}/mod.rs | 413 +--
.../file_format}/parquet.rs | 44 +-
.../file_format}/parquet/metrics.rs | 0
.../file_format}/parquet/page_filter.rs | 6 +-
.../file_format}/parquet/row_filter.rs | 0
.../file_format}/parquet/row_groups.rs | 2 +-
datafusion/core/src/physical_plan/filter.rs | 4 +-
datafusion/core/src/physical_plan/insert.rs | 26 +-
.../core/src/physical_plan/joins/cross_join.rs | 4 +-
.../core/src/physical_plan/joins/hash_join.rs | 269 +-
.../src/physical_plan/joins/hash_join_utils.rs | 80 +-
.../src/physical_plan/joins/nested_loop_join.rs | 16 +-
.../src/physical_plan/joins/sort_merge_join.rs | 22 +-
.../src/physical_plan/joins/symmetric_hash_join.rs | 234 +-
datafusion/core/src/physical_plan/joins/utils.rs | 18 +-
datafusion/core/src/physical_plan/limit.rs | 6 +-
datafusion/core/src/physical_plan/memory.rs | 4 +-
.../core/src/physical_plan/metrics/baseline.rs | 2 +-
datafusion/core/src/physical_plan/mod.rs | 37 +-
.../planner.rs} | 68 +-
datafusion/core/src/physical_plan/projection.rs | 6 +-
.../repartition/distributor_channels.rs | 13 -
.../core/src/physical_plan/repartition/mod.rs | 204 +-
datafusion/core/src/physical_plan/sorts/builder.rs | 2 +-
datafusion/core/src/physical_plan/sorts/merge.rs | 2 +-
datafusion/core/src/physical_plan/sorts/sort.rs | 22 +-
.../physical_plan/sorts/sort_preserving_merge.rs | 49 +-
datafusion/core/src/physical_plan/sorts/stream.rs | 2 +-
datafusion/core/src/physical_plan/stream.rs | 394 +--
datafusion/core/src/physical_plan/streaming.rs | 26 +-
datafusion/core/src/physical_plan/udaf.rs | 57 +-
datafusion/core/src/physical_plan/union.rs | 51 +-
datafusion/core/src/physical_plan/unnest.rs | 7 +-
datafusion/core/src/physical_plan/values.rs | 6 +-
.../windows/bounded_window_agg_exec.rs | 8 +-
datafusion/core/src/physical_plan/windows/mod.rs | 212 +-
.../src/physical_plan/windows/window_agg_exec.rs | 4 +-
datafusion/core/src/prelude.rs | 2 +-
datafusion/core/src/scalar.rs | 7 +-
datafusion/core/src/test/exec.rs | 240 +-
datafusion/core/src/test/mod.rs | 9 +-
datafusion/core/src/test/object_store.rs | 1 -
datafusion/core/src/test_util/mod.rs | 4 +-
datafusion/core/src/test_util/parquet.rs | 3 +-
datafusion/core/src/variable/mod.rs | 2 +-
.../core/tests/{fuzz_cases => }/aggregate_fuzz.rs | 0
datafusion/core/tests/custom_sources.rs | 16 +-
datafusion/core/tests/custom_sources_cases/mod.rs | 19 -
datafusion/core/tests/data/cars.csv | 26 -
.../core/tests/{dataframe/mod.rs => dataframe.rs} | 53 +-
.../tests/{dataframe => }/dataframe_functions.rs | 5 +-
datafusion/core/tests/fifo.rs | 91 -
datafusion/core/tests/fuzz.rs | 26 -
datafusion/core/tests/fuzz_cases/mod.rs | 23 -
.../fuzz_cases/sort_preserving_repartition_fuzz.rs | 237 --
.../core/tests/{fuzz_cases => }/join_fuzz.rs | 0
datafusion/core/tests/memory_limit.rs | 3 +-
.../core/tests/{fuzz_cases => }/merge_fuzz.rs | 0
.../tests/{fuzz_cases => }/order_spill_fuzz.rs | 0
datafusion/core/tests/parquet/custom_reader.rs | 3 +-
datafusion/core/tests/parquet/mod.rs | 9 +-
datafusion/core/tests/parquet/page_pruning.rs | 5 +-
datafusion/core/tests/parquet/schema_coercion.rs | 3 +-
datafusion/core/tests/path_partition.rs | 11 +-
.../provider_filter_pushdown.rs | 4 +-
datafusion/core/tests/{sql => }/repartition.rs | 3 +-
datafusion/core/tests/row.rs | 2 +-
datafusion/core/tests/sql/aggregates.rs | 8 +-
datafusion/core/tests/sql/explain_analyze.rs | 12 +-
datafusion/core/tests/sql/expr.rs | 24 +-
datafusion/core/tests/sql/functions.rs | 143 ++
datafusion/core/tests/sql/information_schema.rs | 10 +-
datafusion/core/tests/sql/joins.rs | 2641 +++++++++++++++++++-
datafusion/core/tests/sql/mod.rs | 539 +++-
datafusion/core/tests/sql/order.rs | 10 +-
datafusion/core/tests/sql/parquet.rs | 6 +-
datafusion/core/tests/sql/parquet_schema.rs | 4 +-
datafusion/core/tests/sql/projection.rs | 6 +-
datafusion/core/tests/sql/subqueries.rs | 732 +++++-
datafusion/core/tests/sql/timestamp.rs | 12 +-
datafusion/core/tests/sql/udf.rs | 43 +-
.../{core_integration.rs => sql_integration.rs} | 3 -
.../sqllogictests/src/engines/datafusion/util.rs | 6 +-
datafusion/core/tests/sqllogictests/src/main.rs | 12 +-
datafusion/core/tests/sqllogictests/src/setup.rs | 174 +-
.../sqllogictests/src/utils.rs} | 33 +-
.../tests/sqllogictests/test_files/aggregate.slt | 110 +-
.../core/tests/sqllogictests/test_files/array.slt | 359 ---
.../core/tests/sqllogictests/test_files/dates.slt | 3 +-
.../tests/sqllogictests/test_files/decimal.slt | 6 -
.../tests/sqllogictests/test_files/explain.slt | 36 -
.../tests/sqllogictests/test_files/functions.slt | 107 -
.../tests/sqllogictests/test_files/groupby.slt | 296 +--
.../core/tests/sqllogictests/test_files/insert.slt | 42 +-
.../tests/sqllogictests/test_files/interval.slt | 8 +-
.../core/tests/sqllogictests/test_files/joins.slt | 2438 ------------------
.../core/tests/sqllogictests/test_files/scalar.slt | 14 +-
.../tests/sqllogictests/test_files/subquery.slt | 868 +------
.../tests/sqllogictests/test_files/timestamps.slt | 211 +-
.../sqllogictests/test_files/tpch/q1.slt.part | 2 +-
.../sqllogictests/test_files/tpch/q10.slt.part | 20 +-
.../sqllogictests/test_files/tpch/q11.slt.part | 157 +-
.../sqllogictests/test_files/tpch/q12.slt.part | 8 +-
.../sqllogictests/test_files/tpch/q13.slt.part | 8 +-
.../sqllogictests/test_files/tpch/q14.slt.part | 6 +-
.../sqllogictests/test_files/tpch/q15.slt.part | 63 +-
.../sqllogictests/test_files/tpch/q16.slt.part | 16 +-
.../sqllogictests/test_files/tpch/q17.slt.part | 20 +-
.../sqllogictests/test_files/tpch/q18.slt.part | 22 +-
.../sqllogictests/test_files/tpch/q19.slt.part | 6 +-
.../sqllogictests/test_files/tpch/q2.slt.part | 60 +-
.../sqllogictests/test_files/tpch/q20.slt.part | 40 +-
.../sqllogictests/test_files/tpch/q21.slt.part | 38 +-
.../sqllogictests/test_files/tpch/q22.slt.part | 75 +-
.../sqllogictests/test_files/tpch/q3.slt.part | 14 +-
.../sqllogictests/test_files/tpch/q4.slt.part | 12 +-
.../sqllogictests/test_files/tpch/q5.slt.part | 32 +-
.../sqllogictests/test_files/tpch/q7.slt.part | 32 +-
.../sqllogictests/test_files/tpch/q8.slt.part | 44 +-
.../sqllogictests/test_files/tpch/q9.slt.part | 32 +-
.../core/tests/sqllogictests/test_files/union.slt | 38 +-
.../core/tests/sqllogictests/test_files/window.slt | 18 +-
.../tests/{custom_sources_cases => }/statistics.rs | 2 +-
datafusion/core/tests/tpcds_planning.rs | 9 +
datafusion/core/tests/user_defined/mod.rs | 25 -
.../tests/user_defined/user_defined_aggregates.rs | 503 ----
.../user_defined/user_defined_window_functions.rs | 548 ----
datafusion/core/tests/user_defined_aggregates.rs | 241 ++
datafusion/core/tests/user_defined_integration.rs | 26 -
.../tests/{user_defined => }/user_defined_plan.rs | 9 +-
.../core/tests/{fuzz_cases => }/window_fuzz.rs | 0
datafusion/execution/Cargo.toml | 4 +-
datafusion/execution/src/registry.rs | 5 +-
datafusion/execution/src/task.rs | 29 +-
datafusion/expr/Cargo.toml | 6 +-
datafusion/expr/src/accumulator.rs | 183 +-
datafusion/expr/src/built_in_function.rs | 776 +-----
datafusion/expr/src/expr.rs | 207 +-
datafusion/expr/src/expr_fn.rs | 137 +-
datafusion/expr/src/expr_rewriter/mod.rs | 110 +-
datafusion/expr/src/expr_schema.rs | 28 +-
datafusion/expr/src/function.rs | 726 +++++-
datafusion/expr/src/function_err.rs | 125 +
datafusion/expr/src/lib.rs | 10 +-
datafusion/expr/src/literal.rs | 61 -
datafusion/expr/src/logical_plan/builder.rs | 18 +-
datafusion/expr/src/logical_plan/plan.rs | 283 ++-
datafusion/expr/src/operator.rs | 24 +-
datafusion/expr/src/partition_evaluator.rs | 284 ---
datafusion/expr/src/signature.rs | 48 +-
datafusion/expr/src/type_coercion/binary.rs | 51 +-
datafusion/expr/src/udaf.rs | 35 +-
datafusion/expr/src/udwf.rs | 118 -
datafusion/expr/src/utils.rs | 28 +-
datafusion/expr/src/window_function.rs | 21 +-
datafusion/optimizer/Cargo.toml | 4 +-
datafusion/{common => optimizer}/src/alias.rs | 1 -
.../optimizer/src/analyzer/count_wildcard_rule.rs | 7 -
datafusion/optimizer/src/analyzer/mod.rs | 2 +-
datafusion/optimizer/src/analyzer/subquery.rs | 12 +-
datafusion/optimizer/src/analyzer/type_coercion.rs | 64 +-
.../optimizer/src/common_subexpr_eliminate.rs | 5 +-
datafusion/optimizer/src/decorrelate.rs | 498 ----
.../src/decorrelate_predicate_subquery.rs | 363 +--
datafusion/optimizer/src/eliminate_outer_join.rs | 7 +-
datafusion/optimizer/src/lib.rs | 2 +-
datafusion/optimizer/src/merge_projection.rs | 52 +-
datafusion/optimizer/src/optimizer.rs | 14 +-
datafusion/optimizer/src/push_down_filter.rs | 3 +-
datafusion/optimizer/src/push_down_projection.rs | 28 +-
.../optimizer/src/rewrite_disjunctive_predicate.rs | 2 +-
.../optimizer/src/scalar_subquery_to_join.rs | 536 ++--
.../src/simplify_expressions/expr_simplifier.rs | 8 +-
.../src/simplify_expressions/simplify_exprs.rs | 20 +-
.../optimizer/src/unwrap_cast_in_comparison.rs | 3 +-
datafusion/optimizer/src/utils.rs | 146 +-
...ptimizer_integration.rs => integration-test.rs} | 22 +-
datafusion/physical-expr/Cargo.toml | 6 +-
.../src/aggregate/array_agg_ordered.rs | 621 -----
datafusion/physical-expr/src/aggregate/average.rs | 24 +-
.../physical-expr/src/aggregate/bit_and_or_xor.rs | 320 +--
.../physical-expr/src/aggregate/bool_and_or.rs | 15 +-
datafusion/physical-expr/src/aggregate/build_in.rs | 77 +-
datafusion/physical-expr/src/aggregate/count.rs | 8 +-
.../physical-expr/src/aggregate/first_last.rs | 203 +-
datafusion/physical-expr/src/aggregate/median.rs | 142 +-
datafusion/physical-expr/src/aggregate/min_max.rs | 16 +-
datafusion/physical-expr/src/aggregate/mod.rs | 20 +-
datafusion/physical-expr/src/aggregate/sum.rs | 8 +-
datafusion/physical-expr/src/aggregate/utils.rs | 32 +-
datafusion/physical-expr/src/array_expressions.rs | 1550 +-----------
.../physical-expr/src/datetime_expressions.rs | 36 +-
datafusion/physical-expr/src/equivalence.rs | 105 +-
datafusion/physical-expr/src/execution_props.rs | 21 +-
datafusion/physical-expr/src/expressions/binary.rs | 81 +-
.../src/expressions/binary/kernels.rs | 372 ++-
datafusion/physical-expr/src/expressions/case.rs | 8 +-
datafusion/physical-expr/src/expressions/cast.rs | 9 -
datafusion/physical-expr/src/expressions/column.rs | 11 -
.../physical-expr/src/expressions/datetime.rs | 8 +-
.../src/expressions/get_indexed_field.rs | 8 +-
.../physical-expr/src/expressions/in_list.rs | 21 +-
.../physical-expr/src/expressions/is_not_null.rs | 8 +-
.../physical-expr/src/expressions/is_null.rs | 8 +-
datafusion/physical-expr/src/expressions/like.rs | 8 +-
.../physical-expr/src/expressions/literal.rs | 8 +-
datafusion/physical-expr/src/expressions/mod.rs | 5 +-
.../physical-expr/src/expressions/negative.rs | 8 +-
datafusion/physical-expr/src/expressions/no_op.rs | 8 +-
datafusion/physical-expr/src/expressions/not.rs | 202 +-
.../physical-expr/src/expressions/try_cast.rs | 8 +-
datafusion/physical-expr/src/functions.rs | 146 +-
datafusion/physical-expr/src/hash_utils.rs | 5 +-
datafusion/physical-expr/src/lib.rs | 2 +
datafusion/physical-expr/src/physical_expr.rs | 39 -
datafusion/physical-expr/src/scalar_function.rs | 21 +-
datafusion/physical-expr/src/sort_expr.rs | 12 +-
datafusion/physical-expr/src/struct_expressions.rs | 2 +-
datafusion/physical-expr/src/type_coercion.rs | 201 ++
.../physical-expr/src/unicode_expressions.rs | 10 +-
datafusion/physical-expr/src/var_provider.rs | 2 +-
datafusion/physical-expr/src/window/aggregate.rs | 14 +-
datafusion/physical-expr/src/window/built_in.rs | 80 +-
.../src/window/built_in_window_function_expr.rs | 51 +-
datafusion/physical-expr/src/window/cume_dist.rs | 14 +-
datafusion/physical-expr/src/window/lead_lag.rs | 34 +-
datafusion/physical-expr/src/window/mod.rs | 4 +
datafusion/physical-expr/src/window/nth_value.rs | 118 +-
datafusion/physical-expr/src/window/ntile.rs | 10 +-
.../src/window/partition_evaluator.rs | 107 +
datafusion/physical-expr/src/window/rank.rs | 45 +-
datafusion/physical-expr/src/window/row_number.rs | 62 +-
.../physical-expr/src/window/sliding_aggregate.rs | 22 +-
datafusion/physical-expr/src/window/window_expr.rs | 140 +-
.../src/window/window_frame_state.rs} | 134 +-
datafusion/proto/Cargo.toml | 2 +-
datafusion/proto/proto/datafusion.proto | 15 -
datafusion/proto/src/bytes/mod.rs | 17 +-
datafusion/proto/src/bytes/registry.rs | 8 +-
datafusion/proto/src/generated/pbjson.rs | 60 -
datafusion/proto/src/generated/prost.rs | 44 -
datafusion/proto/src/logical_plan/from_proto.rs | 83 +-
datafusion/proto/src/logical_plan/mod.rs | 4 +-
datafusion/proto/src/logical_plan/to_proto.rs | 20 -
datafusion/proto/src/physical_plan/from_proto.rs | 3 +-
datafusion/proto/src/physical_plan/mod.rs | 17 +-
datafusion/proto/src/physical_plan/to_proto.rs | 15 +-
datafusion/sql/Cargo.toml | 2 +-
datafusion/sql/examples/sql.rs | 5 -
datafusion/sql/src/expr/arrow_cast.rs | 2 +-
datafusion/sql/src/expr/function.rs | 62 +-
datafusion/sql/src/expr/grouping_set.rs | 4 +-
datafusion/sql/src/expr/identifier.rs | 2 +-
datafusion/sql/src/expr/mod.rs | 27 +-
datafusion/sql/src/expr/unary_op.rs | 8 +-
datafusion/sql/src/expr/value.rs | 84 +-
datafusion/sql/src/parser.rs | 3 +-
datafusion/sql/src/planner.rs | 3 -
datafusion/sql/src/statement.rs | 56 +-
datafusion/sql/src/utils.rs | 13 +-
.../{sql_integration.rs => integration_test.rs} | 84 +-
datafusion/substrait/Cargo.toml | 6 +-
datafusion/substrait/src/logical_plan/consumer.rs | 104 +-
datafusion/substrait/src/logical_plan/producer.rs | 278 +--
datafusion/substrait/src/physical_plan/consumer.rs | 3 +-
datafusion/substrait/src/physical_plan/producer.rs | 2 +-
datafusion/substrait/tests/cases/mod.rs | 20 -
.../tests/cases/roundtrip_logical_plan.rs | 721 ------
.../tests/cases/roundtrip_physical_plan.rs | 77 -
.../substrait/tests/roundtrip_logical_plan.rs | 50 +-
.../substrait/tests/roundtrip_physical_plan.rs | 80 +
.../substrait/tests/{cases => }/serialize.rs | 0
.../substrait/tests/substrait_integration.rs | 19 -
datafusion/substrait/tests/testdata/data.csv | 6 +-
dev/release/rat_exclude_files.txt | 1 -
docs/source/conf.py | 2 +-
docs/source/contributor-guide/roadmap.md | 121 +-
docs/source/user-guide/cli.md | 2 +-
docs/source/user-guide/example-usage.md | 6 +-
docs/source/user-guide/expressions.md | 20 -
docs/source/user-guide/introduction.md | 58 +-
docs/source/user-guide/sql/scalar_functions.md | 305 +--
docs/source/user-guide/sql/sql_status.md | 143 +-
380 files changed, 11523 insertions(+), 20649 deletions(-)
delete mode 100644 datafusion-cli/tests/cli_integration.rs
delete mode 100644 datafusion-cli/tests/data/sql.txt
delete mode 100644 datafusion-examples/examples/simple_udwf.rs
delete mode 100644 datafusion/common/src/display.rs
create mode 100644 datafusion/common/src/from_slice.rs
delete mode 100644 datafusion/common/src/join_type.rs
rename datafusion/core/src/{datasource => }/avro_to_arrow/arrow_array_reader.rs (99%)
rename datafusion/core/src/{datasource => }/avro_to_arrow/mod.rs (92%)
rename datafusion/core/src/{datasource => }/avro_to_arrow/reader.rs (96%)
rename datafusion/core/src/{datasource => }/avro_to_arrow/schema.rs (100%)
copy datafusion/core/src/catalog/{mod.rs => catalog.rs} (97%)
rename datafusion/core/src/datasource/{provider.rs => datasource.rs} (100%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/arrow_file.rs (95%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/avro.rs (95%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/chunked_store.rs (97%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/csv.rs (98%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/file_stream.rs (98%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/json.rs (98%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/mod.rs (81%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/parquet.rs (99%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/parquet/metrics.rs (100%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/parquet/page_filter.rs (99%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/parquet/row_filter.rs (100%)
rename datafusion/core/src/{datasource/physical_plan => physical_plan/file_format}/parquet/row_groups.rs (99%)
rename datafusion/core/src/{physical_planner.rs => physical_plan/planner.rs} (98%)
rename datafusion/core/tests/{fuzz_cases => }/aggregate_fuzz.rs (100%)
delete mode 100644 datafusion/core/tests/custom_sources_cases/mod.rs
delete mode 100644 datafusion/core/tests/data/cars.csv
rename datafusion/core/tests/{dataframe/mod.rs => dataframe.rs} (96%)
rename datafusion/core/tests/{dataframe => }/dataframe_functions.rs (99%)
delete mode 100644 datafusion/core/tests/fuzz.rs
delete mode 100644 datafusion/core/tests/fuzz_cases/mod.rs
delete mode 100644 datafusion/core/tests/fuzz_cases/sort_preserving_repartition_fuzz.rs
rename datafusion/core/tests/{fuzz_cases => }/join_fuzz.rs (100%)
rename datafusion/core/tests/{fuzz_cases => }/merge_fuzz.rs (100%)
rename datafusion/core/tests/{fuzz_cases => }/order_spill_fuzz.rs (100%)
rename datafusion/core/tests/{custom_sources_cases => }/provider_filter_pushdown.rs (98%)
rename datafusion/core/tests/{sql => }/repartition.rs (96%)
create mode 100644 datafusion/core/tests/sql/functions.rs
rename datafusion/core/tests/{core_integration.rs => sql_integration.rs} (92%)
copy datafusion/core/{src/physical_plan/tree_node.rs => tests/sqllogictests/src/utils.rs} (53%)
delete mode 100644 datafusion/core/tests/sqllogictests/test_files/array.slt
rename datafusion/core/tests/{custom_sources_cases => }/statistics.rs (99%)
delete mode 100644 datafusion/core/tests/user_defined/mod.rs
delete mode 100644 datafusion/core/tests/user_defined/user_defined_aggregates.rs
delete mode 100644 datafusion/core/tests/user_defined/user_defined_window_functions.rs
create mode 100644 datafusion/core/tests/user_defined_aggregates.rs
delete mode 100644 datafusion/core/tests/user_defined_integration.rs
rename datafusion/core/tests/{user_defined => }/user_defined_plan.rs (98%)
rename datafusion/core/tests/{fuzz_cases => }/window_fuzz.rs (100%)
create mode 100644 datafusion/expr/src/function_err.rs
delete mode 100644 datafusion/expr/src/partition_evaluator.rs
delete mode 100644 datafusion/expr/src/udwf.rs
rename datafusion/{common => optimizer}/src/alias.rs (98%)
delete mode 100644 datafusion/optimizer/src/decorrelate.rs
rename datafusion/optimizer/tests/{optimizer_integration.rs => integration-test.rs} (96%)
delete mode 100644 datafusion/physical-expr/src/aggregate/array_agg_ordered.rs
create mode 100644 datafusion/physical-expr/src/type_coercion.rs
create mode 100644 datafusion/physical-expr/src/window/partition_evaluator.rs
rename datafusion/{expr/src/window_state.rs => physical-expr/src/window/window_frame_state.rs} (85%)
rename datafusion/sql/tests/{sql_integration.rs => integration_test.rs} (98%)
delete mode 100644 datafusion/substrait/tests/cases/mod.rs
delete mode 100644 datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
delete mode 100644 datafusion/substrait/tests/cases/roundtrip_physical_plan.rs
create mode 100644 datafusion/substrait/tests/roundtrip_physical_plan.rs
rename datafusion/substrait/tests/{cases => }/serialize.rs (100%)
delete mode 100644 datafusion/substrait/tests/substrait_integration.rs