You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/23 11:14:51 UTC
(datafusion-comet) branch dependabot/github_actions/codeql-actions-c83b7ed659 updated (fc62789fdf -> cffed52a1c)
This is an automated email from the ASF dual-hosted git repository.
dependabot[bot] pushed a change to branch dependabot/github_actions/codeql-actions-c83b7ed659
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
omit fc62789fdf build(deps): bump the codeql-actions group across 1 directory with 2 updates
add 53d398bb82 fix: include try_add suggestion in ANSI SUM overflow error (#6732)
add 0f2ae443cd fix: give each nondeterministic dispatched expression its own kernel (#6712)
add 758ef1ceb1 perf: read only the nested fields Spark keeps in the native Iceberg scan (#6725)
add c3316fbe39 refactor: simplify shuffle support predicates in CometShuffleExchangeExec (#7045)
add 86d2d78257 feat: native RANGE window frames with explicit offset on DATE ORDER BY (#4974)
add 6a7e56cddb fix: honor subexpression elimination off in the native null short-circuit (#6986)
add 3a4a159409 fix: read fs.comet.libhdfs.schemes for the scan rule from the Hadoop conf (#6838)
add 1bad7c9e5f perf: make the native Iceberg writer's dictionary pre-pass 1.8x to 3.9x faster (#6937)
add e242944ec4 fix: keep unconverted DPP filters in a scan's canonical form (#6270)
add c4ca20e8b6 fix: fall back to Spark for NOT IN over an ordered streamed side (#6810)
add 46b9c6afe2 fix: stop building maps natively when the key type holds a FLOAT or DOUBLE (#6992)
add d44366230a feat: support PivotFirst aggregate for the optimized PIVOT fast path (#4798)
add f180d65f18 perf: reuse validity buffers and use unary kernels in native expressions (#6953)
add 79ab485ff4 docs: list known Parquet and Iceberg issues and add a data source audit log (#7052)
add a0cbb0fa06 feat: bump iceberg-rust and write float and double identity partitions natively (#6851)
add 27f437a362 refactor: move MergeRows, Explode, aggregate, join and window planning into dedicated modules (#7014)
add 14748af84c fix: return errors instead of panicking in native planner (#7087)
add 642660c2d8 ci: label Delta pull requests with area:Delta (#6965)
add d60cc1dd5f perf: optimize JVM columnar-to-row conversion (#5496)
add 4ef103624e fix: handle NullType struct fields in JVM columnar shuffle (#7099)
add c9f56176cf docs: consolidate Parquet and Iceberg documentation into one page each (#7090)
add 0d0a1c6fe7 fix: size the native runtime for standalone executors without spark.executor.cores (#6368)
add d0f367dc02 test: write ancient dates with CORRECTED rebase mode in the DATE RANGE window test (#7103)
add 4a8a9496d1 fix: fall back to Spark when Iceberg reports an output ordering (#7043)
add 498956132c fix: build release jars from the same commit as the native libraries (#6377)
add aacb2e7441 fix: count grouped integer SUM state in the aggregate's memory reservation (#6370)
add 85b9e39a09 build(deps): bump actions/setup-python from 6 to 7 (#7046)
add 80b518173d refactor: share window frame bound offset serialization between lower and upper bounds (#7107)
add e3503efa6a perf: start bypass shuffle partition writers on small pages that grow (#7059)
add 115058d3cf fix: preserve Parquet root field IDs after nested pruning (#7081)
add 4ab113090d ci: give the Delta contrib Maven cache its own key prefix (#7111)
add 2a2aa05404 perf: write every partition of a JVM shuffle spill in one native call (#7065)
add 0c7c5d69db fix: serialize large-offset string and binary vectors with 32-bit offsets (#6371)
add 3355602e26 fix: keep a streamed side sorted outside the native plan in order through outer hash joins (#6835)
add 47730e38f3 chore: fix stddev accumulator size and resolve session libhdfs schemes once per scan rule (#7120)
add 53d886756b perf: create a native plan's spill directories on its first spill (#7066)
add 3565f2f9b1 fix: treat -0.0 and 0.0 as one value in arrays_overlap on Spark 4.0.5, 4.1.4 and 4.2.1 (#6989)
add cae599ace3 feat: add config to disable or restrict native UDF library loading (#6775)
add 137c894539 feat: support native wide-decimal hashing (#6548)
add 177fef162a refactor: deduplicate split list construction (#7004)
add 99c4873d52 perf: vectorize int, decimal, float and boolean casts to timestamp (#6969)
add 348d38f28f fix: preserve Spark nested null ordering in native sort (#6968)
add d834665aea test: verify native timestamp output timezone labels (#6930)
add 5a8c5c5eda feat: enable native replace for safe inputs (#6680)
add a48c82da53 fix: fall back when an Iceberg equality delete is keyed on a nested field (#7122)
add e4777a6587 fix: fall back for duplicate from_json fields (#6651)
add 39380cca81 feat: opt in to join filter pushdown through deterministic residuals (#6431)
add 0b04b38002 fix: evaluate the array_append item on rows whose array is NULL (#7025)
add 666f86f71b perf: sort JVM shuffle spill pointers in the sorter's scratch space (#7125)
add bf49f1e028 chore: limit open PRs for users without write access to 3 (#7135)
add 9a3d2a292a perf: load the next Parquet file's footer and page index while the current file is read (#7094)
add 74392a9427 test: add window operator benchmarks (#7139)
add 38d392940e perf: default the native Iceberg scan to two concurrent data files (#7132)
add ab884e24ac fix: fall back to Spark for negative-scale decimal casts and arithmetic (#5050)
add da2ff08134 fix: extract bundled native libraries into private directories (#6987)
add 78a0e96f05 fix: preserve Spark AVG buffer and overflow semantics (#5420)
add 5a50bd6f27 test: cover positional struct reconciliation in native CASE WHEN and COALESCE (#7149)
add 58978d0d45 refactor: share Parquet field-id and name resolution between root and nested fields (#6990)
add 50ab6b063c perf: share one decoded copy of a broadcast among the tasks of an executor (#7016)
add 06cff60439 feat: execute scalar Arrow Python UDFs in native Comet (#6130)
add a81a591e58 fix: log native panic backtraces instead of attaching them to the exception (#7042)
add 653c7954a4 feat: support AtLeastNNonNulls natively (#6180)
add 333bae814b fix: report output rows and time of the native window operator (#7140)
add cffed52a1c build(deps): bump the codeql-actions group across 1 directory with 2 updates
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (fc62789fdf)
\
N -- N -- N refs/heads/dependabot/github_actions/codeql-actions-c83b7ed659 (cffed52a1c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.ai/skills/review-comet-iceberg-write-pr/SKILL.md | 13 +-
.asf.yaml | 7 +
.github/actions/rust-test/action.yaml | 23 +
.github/labeler.yml | 13 +
.github/workflows/codeql.yml | 4 +-
.github/workflows/delta_contrib_test.yml | 18 +-
.github/workflows/iceberg_spark_test_reusable.yml | 6 +-
.github/workflows/pr_build_linux.yml | 4 +
.github/workflows/pr_build_macos.yml | 4 +
.github/workflows/pyarrow_udf_test.yml | 36 +-
.../comet/contrib/delta/DeltaScanSupport.scala | 47 +-
.../spark/sql/comet/CometDeltaNativeScanExec.scala | 9 +-
.../contrib/delta/CometDeltaNativeScanSuite.scala | 108 +-
.../contrib/delta/DeltaScanContribSuite.scala | 72 +-
dev/ci/compute-changes.py | 8 +-
dev/local-ci.sh | 2 +
dev/release/build-release-comet.sh | 56 +-
dev/release/comet-rm/build-comet-native-libs.sh | 8 +-
dev/release/rat_exclude_files.txt | 2 -
.../source/_templates/redirect.html | 5 +-
docs/source/about/faq.md | 2 +-
docs/source/about/gluten_comparison.md | 2 +-
docs/source/conf.py | 9 +
.../contributor-guide/adding_a_new_operator.md | 126 +-
.../data-source-audits/iceberg.md | 108 +
.../contributor-guide/data-source-audits/index.md | 43 +
.../data-source-audits/parquet.md | 108 +
docs/source/contributor-guide/debugging.md | 54 +-
docs/source/contributor-guide/development.md | 13 +-
.../expression-audits/agg_funcs.md | 9 +
.../expression-audits/array_funcs.md | 2 +
.../expression-audits/conversion_funcs.md | 1 +
.../expression-audits/hash_funcs.md | 7 +-
.../expression-audits/map_funcs.md | 4 +-
.../expression-audits/predicate_funcs.md | 13 +
.../expression-audits/string_funcs.md | 28 +-
docs/source/contributor-guide/floating_point.md | 14 +-
.../contributor-guide/iceberg-spark-tests.md | 8 +-
docs/source/contributor-guide/iceberg-writes.md | 13 +-
docs/source/contributor-guide/index.md | 1 +
docs/source/contributor-guide/jvm_shuffle.md | 40 +-
docs/source/contributor-guide/memory_management.md | 59 +-
docs/source/contributor-guide/native_shuffle.md | 9 +-
docs/source/contributor-guide/release_process.md | 41 +-
docs/source/contributor-guide/roadmap.md | 2 +-
.../contributor-guide/sql_error_propagation.md | 2 +-
docs/source/contributor-guide/timezones.md | 2 +-
.../expressions/_category_template/cast.md | 25 +
.../latest/compatibility/floating-point.md | 28 +-
.../user-guide/latest/compatibility/index.md | 16 +-
.../user-guide/latest/compatibility/operators.md | 95 +-
.../user-guide/latest/compatibility/scans.md | 152 -
.../latest/compatibility/spark-versions.md | 15 +-
docs/source/user-guide/latest/datasources.md | 27 +-
docs/source/user-guide/latest/datatypes.md | 2 +-
docs/source/user-guide/latest/expressions.md | 22 +-
docs/source/user-guide/latest/iceberg-writes.md | 502 ---
docs/source/user-guide/latest/iceberg.md | 612 +++-
docs/source/user-guide/latest/index.rst | 20 +-
docs/source/user-guide/latest/installation.md | 26 +
docs/source/user-guide/latest/metrics.md | 9 +-
docs/source/user-guide/latest/migration-guide.md | 12 +-
docs/source/user-guide/latest/operators.md | 22 +-
docs/source/user-guide/latest/parquet.md | 359 +++
docs/source/user-guide/latest/pyarrow-udfs.md | 112 +-
docs/source/user-guide/latest/rust_udfs.md | 31 +-
.../user-guide/latest/s3-credential-providers.md | 2 +-
docs/source/user-guide/latest/tuning.md | 31 +-
docs/source/user-guide/latest/tuning/memory.md | 14 +-
docs/source/user-guide/latest/tuning/operators.md | 57 +-
docs/source/user-guide/latest/tuning/scans.md | 72 -
docs/source/user-guide/latest/tuning/shuffle.md | 17 +-
.../source/user-guide/latest/tuning/transitions.md | 30 +
.../user-guide/latest/understanding-comet-plans.md | 2 +-
native/Cargo.lock | 178 +-
native/Cargo.toml | 4 +-
native/common/src/schema.rs | 10 +-
native/core/Cargo.toml | 11 +
native/core/benches/iceberg_dictionary_chooser.rs | 444 +++
native/core/benches/sort_row_partitions.rs | 74 +
native/core/src/comet_native_udf_bridge.rs | 13 +-
native/core/src/execution/c_udf/cache.rs | 220 +-
native/core/src/execution/c_udf/mod.rs | 1 +
native/core/src/execution/c_udf/policy.rs | 351 ++
native/core/src/execution/jni_api.rs | 610 +++-
native/core/src/execution/mod.rs | 5 +-
.../src/execution/operators/arrow_python_udf.rs | 544 ++++
.../execution/operators/dynamic_filter/early.rs | 2 +
.../operators/dynamic_filter/join/tests.rs | 46 +
.../operators/dynamic_filter/parquet_reader.rs | 12 +-
.../src/execution/operators/iceberg_dictionary.rs | 771 ++++-
.../execution/operators/iceberg_partition_value.rs | 13 +-
.../core/src/execution/operators/iceberg_scan.rs | 100 +-
native/core/src/execution/operators/mod.rs | 9 +
.../core/src/execution/operators/sorted_input.rs | 135 +
native/core/src/execution/planner.rs | 3397 +++++---------------
native/core/src/execution/planner/aggregate.rs | 893 +++++
native/core/src/execution/planner/explode.rs | 210 ++
native/core/src/execution/planner/join.rs | 948 ++++++
native/core/src/execution/planner/merge_rows.rs | 235 ++
.../src/execution/planner/operator_registry.rs | 70 +-
native/core/src/execution/planner/window.rs | 775 +++++
native/core/src/execution/python_udf.rs | 656 ++++
native/core/src/execution/row_partition_sort.rs | 264 ++
native/core/src/execution/sort.rs | 225 --
native/core/src/execution/spark_config.rs | 4 +
native/core/src/execution/spill_dirs.rs | 133 +
native/core/src/lib.rs | 10 +-
.../src/parquet/eager_page_index_reader_factory.rs | 632 +++-
native/core/src/parquet/field_resolver.rs | 352 ++
native/core/src/parquet/mod.rs | 1 +
native/core/src/parquet/parquet_exec.rs | 29 +-
native/core/src/parquet/parquet_support.rs | 107 +-
native/core/src/parquet/schema_adapter.rs | 795 ++---
native/jni-bridge/src/errors.rs | 301 +-
native/jni-bridge/testdata/backtrace.txt | 22 -
native/jni-bridge/testdata/stacktrace.txt | 12 -
native/operators/Cargo.toml | 4 +
native/operators/benches/window.rs | 411 +++
native/operators/src/filter.rs | 38 +
native/operators/src/lib.rs | 2 +-
native/operators/src/range.rs | 34 +
native/proto/src/proto/expr.proto | 23 +
native/proto/src/proto/operator.proto | 34 +
native/shuffle/Cargo.toml | 2 +-
native/shuffle/benches/row_columnar.rs | 188 +-
native/shuffle/src/spark_unsafe/row.rs | 592 +++-
native/shuffle/src/writers/checksum.rs | 38 +-
native/spark-expr/Cargo.toml | 8 +
native/spark-expr/benches/at_least_n_non_nulls.rs | 137 +
native/spark-expr/benches/cast_int_to_timestamp.rs | 180 +-
.../benches/cast_non_int_numeric_timestamp.rs | 306 +-
native/spark-expr/benches/decimal_hash.rs | 117 +
native/spark-expr/src/agg_funcs/avg.rs | 36 +-
native/spark-expr/src/agg_funcs/avg_decimal.rs | 482 ++-
native/spark-expr/src/agg_funcs/mod.rs | 2 +
native/spark-expr/src/agg_funcs/pivot_first.rs | 1142 +++++++
native/spark-expr/src/agg_funcs/stddev.rs | 8 +-
native/spark-expr/src/agg_funcs/sum_int.rs | 109 +-
native/spark-expr/src/array_funcs/array_extrema.rs | 8 +-
native/spark-expr/src/array_funcs/array_insert.rs | 10 +-
.../spark-expr/src/array_funcs/arrays_overlap.rs | 320 +-
native/spark-expr/src/array_funcs/mod.rs | 2 +-
native/spark-expr/src/array_funcs/sort_array.rs | 92 +
.../src/bloom_filter/bloom_filter_agg.rs | 11 +-
.../spark-expr/src/bloom_filter/spark_bit_array.rs | 5 +
.../src/bloom_filter/spark_bloom_filter.rs | 5 +
native/spark-expr/src/comet_scalar_funcs.rs | 12 +-
.../spark-expr/src/conditional_funcs/case_when.rs | 260 ++
native/spark-expr/src/conversion_funcs/boolean.rs | 76 +-
native/spark-expr/src/conversion_funcs/numeric.rs | 752 +++--
native/spark-expr/src/conversion_funcs/utils.rs | 20 +-
.../src/datetime_funcs/unix_timestamp.rs | 42 +-
native/spark-expr/src/float_semantics/mod.rs | 32 +
native/spark-expr/src/hash_funcs/utils.rs | 258 +-
native/spark-expr/src/hash_funcs/xxhash64.rs | 4 +-
native/spark-expr/src/hash_funcs/xxhash64_diff.rs | 43 +-
native/spark-expr/src/json_funcs/from_json.rs | 37 +-
native/spark-expr/src/lib.rs | 12 +-
native/spark-expr/src/math_funcs/div.rs | 56 +
native/spark-expr/src/math_funcs/mod.rs | 2 +-
native/spark-expr/src/math_funcs/modulo_expr.rs | 60 +-
.../src/predicate_funcs/at_least_n_non_nulls.rs | 523 +++
native/spark-expr/src/predicate_funcs/mod.rs | 2 +
.../src/string_funcs/regexp_extract_all.rs | 17 +-
native/spark-expr/src/string_funcs/split.rs | 214 +-
.../apache/comet/DirectColumnarToRowConverter.java | 529 +++
.../src/main/java/org/apache/comet/NativeBase.java | 52 +-
.../main/java/org/apache/comet/NativeLibrary.java | 110 +
.../org/apache/comet/udf/CometNativeUdfBridge.java | 9 +-
.../java/org/apache/comet/vector/CometVector.java | 8 +-
.../CometUnboundedShuffleMemoryAllocator.java | 7 +-
.../shuffle/sort/CometShuffleExternalSorter.java | 4 +-
.../org/apache/spark/shuffle/sort/SpillSorter.java | 195 +-
.../shuffle/CometBypassMergeSortShuffleWriter.java | 18 +-
.../execution/shuffle/CometDiskBlockWriter.java | 30 +
.../sql/comet/execution/shuffle/SpillWriter.java | 39 +-
.../main/scala/org/apache/comet/CometConf.scala | 146 +-
.../scala/org/apache/comet/CometExecIterator.scala | 92 +-
.../org/apache/comet/CometKryoRegistrator.scala | 6 +-
spark/src/main/scala/org/apache/comet/Native.scala | 75 +-
.../comet/codegen/CometBatchKernelCodegen.scala | 3 +-
.../codegen/CometBatchKernelCodegenInput.scala | 20 +-
.../apache/comet/codegen/DispatchOccurrence.scala | 64 +
.../org/apache/comet/expressions/CometCast.scala | 57 +-
.../apache/comet/iceberg/IcebergReflection.scala | 284 +-
.../apache/comet/objectstore/NativeConfig.scala | 29 +
.../org/apache/comet/rules/CometExecRule.scala | 11 +-
.../org/apache/comet/rules/CometScanRule.scala | 162 +-
.../org/apache/comet/serde/CometScalaUDF.scala | 8 +-
.../org/apache/comet/serde/CometSortOrder.scala | 51 +-
.../org/apache/comet/serde/QueryPlanSerde.scala | 65 +-
.../scala/org/apache/comet/serde/aggregates.scala | 140 +-
.../scala/org/apache/comet/serde/arithmetic.scala | 58 +-
.../main/scala/org/apache/comet/serde/arrays.scala | 130 +-
.../main/scala/org/apache/comet/serde/hash.scala | 8 +-
.../main/scala/org/apache/comet/serde/maps.scala | 50 +-
.../serde/operator/CometIcebergNativeScan.scala | 129 +-
.../serde/operator/CometIcebergNativeWrite.scala | 28 +-
.../comet/serde/operator/CometNativeScan.scala | 33 +-
.../scala/org/apache/comet/serde/predicates.scala | 36 +-
.../scala/org/apache/comet/serde/strings.scala | 58 +-
.../scala/org/apache/comet/serde/structs.scala | 3 +-
.../org/apache/comet/udf/CometNativeUDF.scala | 28 +-
.../comet/udf/CometNativeUdfExceptions.scala | 6 +
.../comet/udf/codegen/CometScalaUDFCodegen.scala | 15 +-
.../src/main/scala/org/apache/spark/Plugins.scala | 2 +-
.../sql/comet/CometBroadcastExchangeExec.scala | 29 +-
.../spark/sql/comet/CometBroadcastValue.scala | 324 ++
.../spark/sql/comet/CometColumnarToRowExec.scala | 99 +-
.../apache/spark/sql/comet/CometMetricNode.scala | 4 +
.../sql/comet/CometNativeColumnarToRowExec.scala | 2 +-
.../spark/sql/comet/CometNativeScanExec.scala | 14 +-
.../apache/spark/sql/comet/CometSampleExec.scala | 4 +-
.../apache/spark/sql/comet/CometScanUtils.scala | 37 +-
.../sql/comet/CometSubqueryBroadcastExec.scala | 5 +-
.../apache/spark/sql/comet/CometWindowExec.scala | 189 +-
.../sql/comet/CometWindowGroupLimitExec.scala | 4 +-
.../sql/comet/execution/arrow/CachedBatchIpc.scala | 26 +-
.../shuffle/CometShuffleExchangeExec.scala | 18 +-
.../org/apache/spark/sql/comet/operators.scala | 174 +-
.../org/apache/spark/sql/comet/util/Utils.scala | 265 +-
...on.scala => ShimCometArrowEvalPythonExec.scala} | 6 +-
.../shims/ShimCometArrowEvalPythonExec.scala} | 6 +-
.../spark/sql/comet/CometArrowEvalPythonExec.scala | 251 ++
.../shims/ShimCometArrowEvalPythonExec.scala} | 9 +-
.../java/org/apache/comet/NativeLibraryTest.java | 213 ++
.../pyspark/test_native_arrow_udf_files.py | 110 +
.../src/test/resources/pyspark/test_pyarrow_udf.py | 168 +
.../sql-tests/expressions/aggregate/PivotFirst.sql | 571 ++++
.../PivotFirst_collation.sql} | 32 +-
.../sql-tests/expressions/aggregate/avg.sql | 67 +
.../sql-tests/expressions/aggregate/avg_ansi.sql | 56 +
.../expressions/array/array_append_eager_eval.sql | 132 +
.../array/array_null_short_circuit_cse_off.sql | 74 +
.../expressions/array/array_sort_collation.sql | 36 +
.../expressions/array/array_sort_comparator.sql | 4 +-
.../expressions/array/array_sort_default.sql | 69 +
.../expressions/cast/timestamp_labels.sql | 138 +
.../conditional/case_when_field_names.sql | 66 +
.../conditional/coalesce_field_names.sql | 78 +
.../datetime/convert_timestamp_labels.sql | 47 +
.../expressions/datetime/timestamp_labels.sql | 135 +
.../sql-tests/expressions/map/element_at_map.sql | 10 +-
.../sql-tests/expressions/map/get_map_value.sql | 6 +-
.../map/map_builders_float_key_gate.sql | 99 +
.../map/map_builders_float_key_gate_legacy.sql | 41 +
.../map/map_builders_float_key_gate_spark_4_0.sql | 42 +
.../map/map_builders_float_key_last_win.sql | 93 +
.../map/map_from_arrays_dedup_policy.sql | 9 +-
.../map/map_from_entries_dedup_policy.sql | 5 +-
.../map/map_null_short_circuit_cse_off.sql | 48 +
.../round_nondeterministic_child.sql} | 25 +-
.../string/routing_string_opt_in_disabled.sql | 7 +-
.../string/routing_string_opt_in_enabled.sql | 7 +-
.../expressions/string/string_replace.sql | 5 +-
.../struct/from_json_duplicate_fields.sql | 44 +
.../iceberg/metadata_column_partition.sql | 2 +-
.../sql-tests/iceberg/nested_schema_pruning.sql | 136 +
.../sql-tests/join/broadcast_shared_decode.sql | 64 +
.../sql-tests/operators/sort_nested_null_order.sql | 32 +-
.../sql-tests/windows/nested_float_order_keys.sql | 2 +-
.../windows/nested_float_order_keys_strict.sql | 22 +-
.../sql-tests/windows/window_functions.sql | 11 +-
.../approved-plans-v1_4/q49/extended.txt | 2 +-
.../approved-plans-v2_7-spark3_5/q70a/extended.txt | 2 +-
.../approved-plans-v2_7-spark4_1/q14a/extended.txt | 2 +-
.../approved-plans-v2_7/q14a/extended.txt | 2 +-
.../approved-plans-v2_7/q36a/extended.txt | 2 +-
.../approved-plans-v2_7/q49/extended.txt | 2 +-
.../approved-plans-v2_7/q5a/extended.txt | 2 +-
.../approved-plans-v2_7/q70a/extended.txt | 2 +-
.../approved-plans-v2_7/q77a/extended.txt | 2 +-
.../approved-plans-v2_7/q80a/extended.txt | 2 +-
.../approved-plans-v2_7/q86a/extended.txt | 2 +-
.../apache/comet/CometArrayExpressionSuite.scala | 20 +-
.../scala/org/apache/comet/CometCodegenSuite.scala | 228 +-
.../org/apache/comet/CometExpressionSuite.scala | 317 +-
.../org/apache/comet/CometFuzzAggregateSuite.scala | 28 +-
.../org/apache/comet/CometFuzzIcebergSuite.scala | 10 +-
.../org/apache/comet/CometFuzzTestSuite.scala | 20 +-
.../apache/comet/CometHashExpressionSuite.scala | 71 +-
.../org/apache/comet/CometIcebergNativeSuite.scala | 814 ++++-
.../org/apache/comet/CometIcebergTestBase.scala | 30 +
.../comet/CometIcebergWriteActionSuite.scala | 184 +-
.../comet/CometIcebergWriteDetectionSuite.scala | 89 +-
.../org/apache/comet/CometMapExpressionSuite.scala | 6 +-
.../org/apache/comet/CometNativeCastSuite.scala | 202 +-
.../scala/org/apache/comet/CometNativeSuite.scala | 71 +-
.../org/apache/comet/CometNativeUdfSuite.scala | 70 +-
.../comet/DirectColumnarToRowConverterSuite.scala | 320 ++
.../apache/comet/exec/CometAggregateSuite.scala | 474 ++-
.../comet/exec/CometColumnarShuffleSuite.scala | 37 +-
.../comet/exec/CometDirectColumnarToRowSuite.scala | 103 +
.../org/apache/comet/exec/CometExecSuite.scala | 152 +-
.../comet/exec/CometInMemoryCacheKryoSuite.scala | 10 +-
.../comet/exec/CometInMemoryCacheSuite.scala | 267 +-
.../org/apache/comet/exec/CometJoinSuite.scala | 960 +++++-
.../comet/exec/CometNativeShuffleSuite.scala | 106 +-
.../exec/CometShuffleInputConversionSuite.scala | 46 +-
.../apache/comet/exec/CometTypedDatasetSuite.scala | 43 +-
.../apache/comet/exec/CometWindowExecSuite.scala | 353 +-
.../comet/objectstore/NativeConfigSuite.scala | 25 +
.../apache/comet/parquet/ParquetReadSuite.scala | 98 +-
.../apache/comet/rules/CometExecRuleSuite.scala | 30 +-
.../comet/rules/CometScanSchemeFallbackSuite.scala | 140 +-
.../apache/comet/serde/QueryPlanSerdeSuite.scala | 62 +-
.../serde/operator/CometNativeScanSuite.scala | 63 +-
.../apache/iceberg/data/CometEqualityDeletes.scala | 8 +-
.../spark/CometExecIteratorLifecycleSuite.scala | 33 +
.../spark/shuffle/sort/SpillSorterSuite.scala | 168 +-
.../scala/org/apache/spark/sql/CometTestBase.scala | 96 +-
.../spark/sql/benchmark/CometBenchmarkBase.scala | 2 +-
.../sql/benchmark/CometC2RIsolatedBench.scala | 217 +-
.../CometColumnarShuffleWriteBenchmark.scala | 126 +
.../benchmark/CometHashExpressionBenchmark.scala | 63 +
.../sql/benchmark/CometIcebergReadBenchmark.scala | 109 +-
.../spark/sql/benchmark/CometPivotBenchmark.scala | 141 +
.../CometPredicateExpressionBenchmark.scala | 98 +-
.../sql/benchmark/CometShuffleBenchmark.scala | 7 +-
.../benchmark/CometStringExpressionBenchmark.scala | 85 +-
.../spark/sql/benchmark/CometWindowBenchmark.scala | 208 ++
.../spark/sql/comet/CometBroadcastValueSuite.scala | 462 +++
.../CometCelebornShufflePlanningSuite.scala | 59 +-
.../shuffle/CometDiskBlockWriterSuite.scala | 195 +-
.../apache/spark/sql/comet/util/UtilsSuite.scala | 279 +-
.../benchmark/CometArrowPythonUdfBenchmark.scala | 133 +
.../spark/sql/comet/CometArrowPythonUdfSuite.scala | 556 ++++
328 files changed, 31702 insertions(+), 7653 deletions(-)
copy native/proto/README.md => docs/source/_templates/redirect.html (72%)
create mode 100644 docs/source/contributor-guide/data-source-audits/iceberg.md
create mode 100644 docs/source/contributor-guide/data-source-audits/index.md
create mode 100644 docs/source/contributor-guide/data-source-audits/parquet.md
delete mode 100644 docs/source/user-guide/latest/compatibility/scans.md
delete mode 100644 docs/source/user-guide/latest/iceberg-writes.md
create mode 100644 docs/source/user-guide/latest/parquet.md
delete mode 100644 docs/source/user-guide/latest/tuning/scans.md
create mode 100644 native/core/benches/iceberg_dictionary_chooser.rs
create mode 100644 native/core/benches/sort_row_partitions.rs
create mode 100644 native/core/src/execution/c_udf/policy.rs
create mode 100644 native/core/src/execution/operators/arrow_python_udf.rs
create mode 100644 native/core/src/execution/operators/sorted_input.rs
create mode 100644 native/core/src/execution/planner/aggregate.rs
create mode 100644 native/core/src/execution/planner/explode.rs
create mode 100644 native/core/src/execution/planner/join.rs
create mode 100644 native/core/src/execution/planner/merge_rows.rs
create mode 100644 native/core/src/execution/planner/window.rs
create mode 100644 native/core/src/execution/python_udf.rs
create mode 100644 native/core/src/execution/row_partition_sort.rs
delete mode 100644 native/core/src/execution/sort.rs
create mode 100644 native/core/src/execution/spill_dirs.rs
create mode 100644 native/core/src/parquet/field_resolver.rs
delete mode 100644 native/jni-bridge/testdata/backtrace.txt
delete mode 100644 native/jni-bridge/testdata/stacktrace.txt
create mode 100644 native/operators/benches/window.rs
create mode 100644 native/spark-expr/benches/at_least_n_non_nulls.rs
create mode 100644 native/spark-expr/benches/decimal_hash.rs
create mode 100644 native/spark-expr/src/agg_funcs/pivot_first.rs
create mode 100644 native/spark-expr/src/predicate_funcs/at_least_n_non_nulls.rs
create mode 100644 spark/src/main/java/org/apache/comet/DirectColumnarToRowConverter.java
create mode 100644 spark/src/main/java/org/apache/comet/NativeLibrary.java
create mode 100644 spark/src/main/scala/org/apache/comet/codegen/DispatchOccurrence.scala
create mode 100644 spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastValue.scala
copy spark/src/main/spark-3.x/org/apache/spark/sql/comet/shims/{ShimCometEmptyRelation.scala => ShimCometArrowEvalPythonExec.scala} (84%)
copy spark/src/main/{spark-3.x/org/apache/spark/sql/comet/shims/ShimCometEmptyRelation.scala => spark-4.0/org/apache/spark/sql/comet/shims/ShimCometArrowEvalPythonExec.scala} (84%)
create mode 100644 spark/src/main/spark-4.1+/org/apache/spark/sql/comet/CometArrowEvalPythonExec.scala
copy spark/src/main/{spark-3.x/org/apache/spark/sql/comet/shims/ShimCometEmptyRelation.scala => spark-4.1+/org/apache/spark/sql/comet/shims/ShimCometArrowEvalPythonExec.scala} (71%)
create mode 100644 spark/src/test/java/org/apache/comet/NativeLibraryTest.java
create mode 100644 spark/src/test/resources/pyspark/test_native_arrow_udf_files.py
create mode 100644 spark/src/test/resources/sql-tests/expressions/aggregate/PivotFirst.sql
copy spark/src/test/resources/sql-tests/expressions/{conditional/nullif_zeroif.sql => aggregate/PivotFirst_collation.sql} (53%)
create mode 100644 spark/src/test/resources/sql-tests/expressions/aggregate/avg_ansi.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/array/array_append_eager_eval.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit_cse_off.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/array/array_sort_collation.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/array/array_sort_default.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/cast/timestamp_labels.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/convert_timestamp_labels.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/timestamp_labels.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/map/map_builders_float_key_gate.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/map/map_builders_float_key_gate_legacy.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/map/map_builders_float_key_gate_spark_4_0.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/map/map_builders_float_key_last_win.sql
create mode 100644 spark/src/test/resources/sql-tests/expressions/map/map_null_short_circuit_cse_off.sql
copy spark/src/test/resources/sql-tests/expressions/{map/element_at_map_collation.sql => math/round_nondeterministic_child.sql} (50%)
create mode 100644 spark/src/test/resources/sql-tests/expressions/struct/from_json_duplicate_fields.sql
create mode 100644 spark/src/test/resources/sql-tests/iceberg/nested_schema_pruning.sql
create mode 100644 spark/src/test/resources/sql-tests/join/broadcast_shared_decode.sql
create mode 100644 spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala
create mode 100644 spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala
create mode 100644 spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarShuffleWriteBenchmark.scala
create mode 100644 spark/src/test/scala/org/apache/spark/sql/benchmark/CometPivotBenchmark.scala
create mode 100644 spark/src/test/scala/org/apache/spark/sql/benchmark/CometWindowBenchmark.scala
create mode 100644 spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala
create mode 100644 spark/src/test/spark-4.1+/org/apache/spark/sql/benchmark/CometArrowPythonUdfBenchmark.scala
create mode 100644 spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org