You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2023/05/19 11:05:07 UTC
[arrow-datafusion] branch dependabot/cargo/main/async-compression-0.4.0 updated (c7ce31f313 -> 07d3e86119)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a change to branch dependabot/cargo/main/async-compression-0.4.0
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
from c7ce31f313 chore(deps): update async-compression requirement from 0.3.14 to 0.4.0
add a07d6ebc9f [parquet] Avoid read parquet index when there is no filter pushdown. (#6317)
add 68061aaccc Parallel merge sort (#6162) (#6308)
add 85bf2afdbc Port some tests in joins.rs to sqllogictest (#6306)
add 7696aa52df Fix case evaluation with NULL (#6334)
add b9e5c071ec Support null values in Avro string columns (#6307)
add d6176d5a42 Add bdt to the list of known users (#6341)
add 2c1161abde port errors.rs test to context.rs (#6340)
add 482f32caa9 Enable parquet page level skipping (page index pruning) by default (#5099)
add cbdf4cda45 Bump snmalloc version in the example usage doc (#6344)
add 9ae370c580 Port tests in explain.rs to sqllogictests (#6343)
add d0aadd6bae Improve parallelism of repartition operator (#6310)
add 4b21b61559 Ordering satisfy consider ordering equivalence of different lengths (#6330)
add 90775b4832 Minor: Update documentation for `datafusion.execution.parquet.enable_page_index` (#6342)
add 063f99fd61 refine decimal multiply, avoid cast to wider type (#6331)
add ca7f760cc9 Combine the two rules: DecorrelateWhereExists and DecorrelateWhereIn (#6271)
add 93ff57e6e0 feat: support bitwise and boolean aggregate functions (#6276)
add 62621eef47 Fix explain plan formatting in sqllogictest (#6329)
add eb918ab217 Fix expected output (#6353)
add 7563cdbbcd Simplify IsNotNull and IsNull expression (#6345)
add d8a92be182 Add support for ordering sensitive aggregation (#6332)
add b578c5819f chore(deps): update substrait requirement from 0.9.0 to 0.10.0 (#6351)
add 33b15c1e8a Minor: Update the testing section of contributor guide (#6357)
add 8a47c42096 Add support for reading Arrow files (#6337)
add 20c7af3bfe Support CREATE TABLE via SQL for infinite streams (#6352)
add 5c40142722 feat: min/max agg for bool (#6226)
add 301e31bec2 Fix shadowing in test_code (#6361)
add cf811170bb Cleanup ExternalSorter metrics (#5885) (#6364)
add 19b0324092 Fix nullability calculation for boolean expressions (#6365)
add 9f808f4c81 Faster ListingTable partition listing (#6182) (#6183)
add 3e3e9b5545 feat: add pattern for simplifying exprs like `str ~ '^foo$'` (#6369)
add 7e5c92db5f Minor: remove left over println (#6375)
add c24830a898 Limit the number of partition files to be displayed for FileGroupsDisplay (#6359)
add 7c6b41a7c7 Switch to non-recursive on heap virtual stack when building logical plan from SQL expression (#6360)
add 0536219375 Fix UNION ALL aliasing (#6373)
add d01002c332 [sqllogictest] port tests in avro.rs to sqllogictest (#6362)
add 4b8a3d6969 Parallel parquet schema inference (#6366)
add cb3f356312 Merge remote-tracking branch 'apache/main' into dependabot/cargo/main/async-compression-0.4.0
add 07d3e86119 Update datafusion-cli lock
No new revisions were added by this update.
Summary of changes:
.github/workflows/rust.yml | 2 +-
benchmarks/expected-plans/q16.txt | 2 +-
benchmarks/expected-plans/q18.txt | 2 +-
benchmarks/expected-plans/q20.txt | 4 +-
benchmarks/expected-plans/q21.txt | 18 +-
benchmarks/expected-plans/q22.txt | 5 +-
benchmarks/expected-plans/q4.txt | 9 +-
datafusion-cli/Cargo.lock | 286 +++---
datafusion/common/src/config.rs | 7 +-
datafusion/common/src/scalar.rs | 108 +++
.../core/src/avro_to_arrow/arrow_array_reader.rs | 27 +-
datafusion/core/src/catalog/listing_schema.rs | 1 +
.../core/src/datasource/file_format/arrow.rs | 96 ++
datafusion/core/src/datasource/file_format/avro.rs | 85 ++
.../core/src/datasource/file_format/file_type.rs | 7 +-
datafusion/core/src/datasource/file_format/mod.rs | 1 +
.../core/src/datasource/file_format/options.rs | 69 ++
.../core/src/datasource/file_format/parquet.rs | 13 +-
datafusion/core/src/datasource/listing/helpers.rs | 514 +++++-----
datafusion/core/src/datasource/listing/table.rs | 5 +-
datafusion/core/src/datasource/listing/url.rs | 46 +-
.../core/src/datasource/listing_table_factory.rs | 13 +-
datafusion/core/src/execution/context.rs | 74 ++
.../src/physical_optimizer/aggregate_statistics.rs | 12 +
.../combine_partial_final_agg.rs | 4 +
.../src/physical_optimizer/dist_enforcement.rs | 11 +-
.../core/src/physical_optimizer/repartition.rs | 2 +
.../src/physical_optimizer/sort_enforcement.rs | 1 +
.../core/src/physical_optimizer/sort_pushdown.rs | 2 +-
.../core/src/physical_plan/aggregates/mod.rs | 193 +++-
datafusion/core/src/physical_plan/common.rs | 26 +
.../src/physical_plan/file_format/arrow_file.rs | 166 ++++
.../core/src/physical_plan/file_format/mod.rs | 22 +-
.../core/src/physical_plan/file_format/parquet.rs | 17 +-
.../file_format/parquet/page_filter.rs | 5 +
.../core/src/physical_plan/metrics/baseline.rs | 50 +-
.../core/src/physical_plan/metrics/composite.rs | 205 ----
datafusion/core/src/physical_plan/metrics/mod.rs | 4 -
.../core/src/physical_plan/metrics/tracker.rs | 104 --
datafusion/core/src/physical_plan/planner.rs | 151 ++-
.../core/src/physical_plan/repartition/mod.rs | 34 +-
datafusion/core/src/physical_plan/sorts/merge.rs | 26 +-
datafusion/core/src/physical_plan/sorts/sort.rs | 223 ++---
.../physical_plan/sorts/sort_preserving_merge.rs | 50 +-
datafusion/core/src/physical_plan/windows/mod.rs | 2 +-
datafusion/core/tests/aggregate_fuzz.rs | 2 +
datafusion/core/tests/data/example.arrow | Bin 0 -> 1842 bytes
datafusion/core/tests/parquet/page_pruning.rs | 39 +
datafusion/core/tests/path_partition.rs | 83 +-
datafusion/core/tests/sql/arrow_files.rs | 70 ++
datafusion/core/tests/sql/avro.rs | 157 ---
datafusion/core/tests/sql/errors.rs | 54 --
datafusion/core/tests/sql/explain.rs | 62 --
datafusion/core/tests/sql/joins.rs | 632 +++---------
datafusion/core/tests/sql/mod.rs | 98 +-
datafusion/core/tests/sql/subqueries.rs | 34 +-
.../src/engines/datafusion/normalize.rs | 15 +-
datafusion/core/tests/sqllogictests/src/main.rs | 57 +-
datafusion/core/tests/sqllogictests/src/setup.rs | 36 +-
.../tests/sqllogictests/test_files/aggregate.slt | 126 +++
.../core/tests/sqllogictests/test_files/avro.slt | 97 ++
.../test_files/create_external_table.slt | 4 +
.../core/tests/sqllogictests/test_files/ddl.slt | 33 +-
.../core/tests/sqllogictests/test_files/errors.slt | 1 -
.../tests/sqllogictests/test_files/explain.slt | 119 ++-
.../tests/sqllogictests/test_files/groupby.slt | 224 ++++-
.../test_files/information_schema.slt | 2 +-
.../core/tests/sqllogictests/test_files/joins.slt | 624 ++++++++++++
.../core/tests/sqllogictests/test_files/json.slt | 10 +-
.../core/tests/sqllogictests/test_files/order.slt | 8 +-
.../test_files/pg_compat/pg_compat_simple.slt | 26 +-
.../test_files/pg_compat/pg_compat_window.slt | 828 ++++++++--------
.../core/tests/sqllogictests/test_files/select.slt | 12 +-
.../core/tests/sqllogictests/test_files/union.slt | 247 ++---
.../core/tests/sqllogictests/test_files/window.slt | 1025 ++++++++++----------
datafusion/execution/src/disk_manager.rs | 4 +-
datafusion/expr/src/aggregate_function.rs | 27 +
datafusion/expr/src/expr.rs | 60 +-
datafusion/expr/src/expr_fn.rs | 12 +
datafusion/expr/src/expr_schema.rs | 24 +-
datafusion/expr/src/logical_plan/builder.rs | 11 +-
datafusion/expr/src/logical_plan/ddl.rs | 3 +
datafusion/expr/src/tree_node/expr.rs | 37 +-
datafusion/expr/src/type_coercion/aggregates.rs | 65 +-
datafusion/expr/src/type_coercion/binary.rs | 21 +-
datafusion/expr/src/udaf.rs | 1 +
.../optimizer/src/analyzer/count_wildcard_rule.rs | 2 +
datafusion/optimizer/src/analyzer/type_coercion.rs | 20 +-
.../optimizer/src/common_subexpr_eliminate.rs | 1 +
...ere_in.rs => decorrelate_predicate_subquery.rs} | 979 ++++++++++++++++---
.../optimizer/src/decorrelate_where_exists.rs | 781 ---------------
datafusion/optimizer/src/lib.rs | 3 +-
datafusion/optimizer/src/optimizer.rs | 6 +-
datafusion/optimizer/src/push_down_projection.rs | 1 +
.../src/simplify_expressions/expr_simplifier.rs | 68 ++
.../optimizer/src/simplify_expressions/regex.rs | 60 +-
.../src/simplify_expressions/simplify_exprs.rs | 35 +-
.../optimizer/src/single_distinct_to_groupby.rs | 3 +
datafusion/optimizer/src/utils.rs | 2 +-
datafusion/optimizer/tests/integration-test.rs | 24 +-
.../physical-expr/src/aggregate/bit_and_or_xor.rs | 934 ++++++++++++++++++
.../physical-expr/src/aggregate/bool_and_or.rs | 582 +++++++++++
datafusion/physical-expr/src/aggregate/build_in.rs | 150 ++-
datafusion/physical-expr/src/aggregate/min_max.rs | 90 +-
datafusion/physical-expr/src/aggregate/mod.rs | 2 +
.../physical-expr/src/aggregate/row_accumulator.rs | 3 +-
datafusion/physical-expr/src/equivalence.rs | 8 +-
.../src/expressions/binary/kernels_arrow.rs | 22 +-
datafusion/physical-expr/src/expressions/case.rs | 35 +-
datafusion/physical-expr/src/expressions/mod.rs | 4 +-
datafusion/physical-expr/src/sort_expr.rs | 45 +-
datafusion/physical-expr/src/utils.rs | 384 +++++---
datafusion/proto/proto/datafusion.proto | 13 +
datafusion/proto/src/generated/pbjson.rs | 178 ++++
datafusion/proto/src/generated/prost.rs | 29 +
datafusion/proto/src/logical_plan/from_proto.rs | 21 +
datafusion/proto/src/logical_plan/mod.rs | 7 +
datafusion/proto/src/logical_plan/to_proto.rs | 32 +-
datafusion/proto/src/physical_plan/from_proto.rs | 25 +
datafusion/proto/src/physical_plan/mod.rs | 33 +-
datafusion/proto/src/physical_plan/to_proto.rs | 43 +-
datafusion/row/src/accessor.rs | 56 ++
datafusion/sql/Cargo.toml | 1 +
datafusion/sql/src/expr/binary_op.rs | 27 +-
datafusion/sql/src/expr/function.rs | 6 +-
datafusion/sql/src/expr/mod.rs | 210 +++-
datafusion/sql/src/parser.rs | 162 +++-
datafusion/sql/src/statement.rs | 2 +
datafusion/sql/src/utils.rs | 24 +-
datafusion/substrait/Cargo.toml | 2 +-
datafusion/substrait/src/logical_plan/consumer.rs | 4 +
datafusion/substrait/src/logical_plan/producer.rs | 3 +-
docs/source/contributor-guide/index.md | 45 +-
docs/source/user-guide/configs.md | 2 +-
docs/source/user-guide/example-usage.md | 2 +-
docs/source/user-guide/expressions.md | 5 +
docs/source/user-guide/introduction.md | 1 +
docs/source/user-guide/sql/aggregate_functions.md | 74 +-
docs/source/user-guide/sql/select.md | 9 +
testing | 2 +-
140 files changed, 8504 insertions(+), 4452 deletions(-)
create mode 100644 datafusion/core/src/datasource/file_format/arrow.rs
create mode 100644 datafusion/core/src/physical_plan/file_format/arrow_file.rs
delete mode 100644 datafusion/core/src/physical_plan/metrics/composite.rs
delete mode 100644 datafusion/core/src/physical_plan/metrics/tracker.rs
create mode 100644 datafusion/core/tests/data/example.arrow
create mode 100644 datafusion/core/tests/sql/arrow_files.rs
delete mode 100644 datafusion/core/tests/sql/avro.rs
delete mode 100644 datafusion/core/tests/sql/errors.rs
delete mode 100644 datafusion/core/tests/sql/explain.rs
create mode 100644 datafusion/core/tests/sqllogictests/test_files/avro.slt
create mode 100644 datafusion/core/tests/sqllogictests/test_files/joins.slt
rename datafusion/optimizer/src/{decorrelate_where_in.rs => decorrelate_predicate_subquery.rs} (52%)
delete mode 100644 datafusion/optimizer/src/decorrelate_where_exists.rs
create mode 100644 datafusion/physical-expr/src/aggregate/bit_and_or_xor.rs
create mode 100644 datafusion/physical-expr/src/aggregate/bool_and_or.rs