You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/09/05 14:23:27 UTC
(datafusion-comet) branch df55 updated (f536ca7f9b -> 12540deac2)
This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a change to branch df55
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
from f536ca7f9b fix: update Celeborn map-key assertion for Arrow 59.2 (#5697)
add e2ded76bb2 fix(celeborn): reject unsafe native push completion tracking (#5665)
add 4219fc7935 fix: match Spark's null short-circuiting in array_join and enable it natively (#5558)
add c052ea4686 fix: make native shuffle spill metrics independent of input batching (#5628)
add 55ae4f206e fix: expand object store option references, uniquify constant metadata names, drop dead parquet JNI (#5653)
add eaf6426a97 feat: Support Iceberg system functions (bucket, truncate, years/months/days/hours) natively (#5638)
add 9cac7164a4 test: exercise the Iceberg write split-operator plan in Iceberg's own suites (#5640)
add 81d637b9bf feat: implement regr_slope, regr_intercept, regr_r2, regr_sxx, regr_syy, regr_sxy aggregates (#4775)
add 683a021942 fix: match Spark's ANSI bound check for float/double to integral casts (#5683)
add a1352ca070 fix: return NULL from rpad/lpad when the length column is NULL instead of panicking (#5680)
add 165e2c67d1 feat: carry VariantType identity through schema serialization (#5631)
add 1119debb36 perf: reuse per-partition scratch in the shuffle write path (#5568)
add 72fcc6f940 chore(deps): bump the codeql-actions group with 2 updates (#5669)
add 296470cab6 fix: fall back for concat_ws with array arguments instead of failing natively (#5679)
add 2da32915c8 fix: prevent silent overflow when reading Parquet TIMESTAMP_MILLIS values (#5177)
add 719cba1107 test: add explode operator microbenchmark (#5381)
add 93c323466f feat: route unrecognized StaticInvoke and Invoke through the codegen dispatcher (#5692)
add 12540deac2 Merge remote-tracking branch 'apache/main' into df55
No new revisions were added by this update.
Summary of changes:
.github/workflows/codeql.yml | 4 +-
.github/workflows/pr_build_linux.yml | 3 +
.github/workflows/pr_build_macos.yml | 3 +
dev/diffs/iceberg/1.10.0.diff | 449 +++++++------
dev/diffs/iceberg/1.11.0.diff | 231 ++++---
dev/diffs/iceberg/1.8.1.diff | 453 +++++++------
dev/diffs/iceberg/1.9.1.diff | 453 +++++++------
.../expression-audits/array_funcs.md | 3 +-
.../expression-audits/string_funcs.md | 4 +-
.../contributor-guide/iceberg-spark-tests.md | 5 +
docs/source/user-guide/latest/expressions.md | 20 +-
docs/source/user-guide/latest/iceberg-writes.md | 5 +-
docs/source/user-guide/latest/iceberg.md | 35 +
docs/source/user-guide/latest/metrics.md | 8 +
docs/source/user-guide/latest/tuning.md | 10 +-
.../core/src/execution/operators/iceberg_write.rs | 359 +++++++++++
native/core/src/execution/planner.rs | 71 ++-
native/core/src/execution/serde.rs | 83 ++-
native/core/src/parquet/cast_column.rs | 306 +++------
native/core/src/parquet/mod.rs | 306 ---------
native/core/src/parquet/parquet_exec.rs | 11 +-
native/core/src/parquet/parquet_support.rs | 131 +++-
native/core/src/parquet/schema_adapter.rs | 8 +-
native/proto/src/proto/expr.proto | 31 +
native/proto/src/proto/operator.proto | 3 +
native/proto/src/proto/types.proto | 1 +
native/shuffle/benches/shuffle_writer.rs | 29 +
native/shuffle/src/metrics.rs | 3 +-
native/shuffle/src/partitioners/multi_partition.rs | 364 ++++++++---
.../src/partitioners/partitioned_batch_iterator.rs | 139 +++-
native/shuffle/src/shuffle_writer.rs | 124 +++-
native/shuffle/src/writers/buf_batch_writer.rs | 225 ++++++-
.../src/writers/local/local_partition_writer.rs | 127 +++-
native/shuffle/src/writers/local/spill.rs | 76 ++-
native/spark-expr/Cargo.toml | 6 +-
native/spark-expr/benches/iceberg_transforms.rs | 218 +++++++
native/spark-expr/src/agg_funcs/mod.rs | 2 +
native/spark-expr/src/agg_funcs/regr.rs | 706 +++++++++++++++++++++
native/spark-expr/src/comet_scalar_funcs.rs | 17 +-
native/spark-expr/src/conversion_funcs/numeric.rs | 361 ++++++++++-
.../src/datetime_funcs/date_from_unix_date.rs | 40 +-
native/spark-expr/src/iceberg_funcs/bucket.rs | 371 +++++++++++
native/spark-expr/src/iceberg_funcs/mod.rs | 139 ++++
native/spark-expr/src/iceberg_funcs/temporal.rs | 408 ++++++++++++
native/spark-expr/src/iceberg_funcs/truncate.rs | 427 +++++++++++++
native/spark-expr/src/lib.rs | 2 +
.../char_varchar_utils/read_side_padding.rs | 112 +++-
native/spark-expr/src/utils.rs | 9 +-
.../shuffle/CelebornShufflePartitionPusher.java | 16 +-
.../shuffle/CelebornTransportCallbackTracker.java | 69 +-
.../comet/NativeColumnarToRowConverter.scala | 3 +-
.../apache/comet/objectstore/NativeConfig.scala | 29 +-
.../org/apache/comet/rules/CometExecRule.scala | 21 +-
.../org/apache/comet/serde/CometScalaUDF.scala | 37 +-
.../org/apache/comet/serde/QueryPlanSerde.scala | 15 +-
.../scala/org/apache/comet/serde/aggregates.scala | 131 +++-
.../main/scala/org/apache/comet/serde/arrays.scala | 66 +-
.../org/apache/comet/serde/icebergFunctions.scala | 263 ++++++++
.../org/apache/comet/serde/namedExpressions.scala | 9 +-
.../comet/serde/operator/CometNativeScan.scala | 56 +-
.../scala/org/apache/comet/serde/statics.scala | 94 ++-
.../scala/org/apache/comet/serde/strings.scala | 24 +-
.../spark/sql/comet/CometNativeScanExec.scala | 1 +
.../shuffle/CometCelebornShuffleManager.scala | 10 +-
.../org/apache/spark/sql/comet/util/Utils.scala | 19 +-
.../org/apache/comet/shims/CometTypeShim.scala | 6 +
.../org/apache/comet/shims/CometTypeShim.scala | 11 +
.../CelebornReflectionCompatibilitySuite.scala | 18 +
.../CelebornTransportOwnershipTestHelper.java | 8 +-
.../sql-tests/expressions/aggregate/regr.sql | 85 ++-
.../sql-tests/expressions/array/array_join.sql | 72 ++-
.../array/array_join_eager_eval_dispatch.sql | 48 ++
...parator.sql => array_join_null_replacement.sql} | 32 +-
.../sql-tests/expressions/datetime/to_time.sql | 10 +-
.../sql-tests/expressions/misc/variant.sql | 4 +
.../sql-tests/expressions/string/concat_ws.sql | 44 +-
.../sql-tests/expressions/string/string_lpad.sql | 6 +-
.../sql-tests/expressions/string/string_rpad.sql | 6 +-
.../apache/comet/CometArrayExpressionSuite.scala | 105 ++-
.../scala/org/apache/comet/CometCodegenSuite.scala | 81 ++-
.../comet/CometIcebergSystemFunctionSuite.scala | 481 ++++++++++++++
.../org/apache/comet/CometIcebergTestBase.scala | 25 +
.../comet/CometIcebergWriteActionSuite.scala | 32 +-
.../org/apache/comet/CometNativeCastSuite.scala | 161 +++++
.../apache/comet/CometStringExpressionSuite.scala | 89 ++-
.../org/apache/comet/CometVariantTypeSuite.scala | 96 +++
.../comet/objectstore/NativeConfigSuite.scala | 29 +
.../apache/comet/parquet/ParquetReadSuite.scala | 216 +++++++
.../serde/operator/CometNativeScanSuite.scala | 51 ++
.../CelebornShufflePartitionPusherSuite.scala | 68 +-
.../org/apache/comet/vector/NativeUtilSuite.scala | 77 +++
.../spark/sql/benchmark/CometBenchmarkBase.scala | 2 +-
.../benchmark/CometCodegenDispatchBenchmark.scala | 407 ++++++++++++
.../sql/benchmark/CometExplodeBenchmark.scala | 428 +++++++++++++
.../CometIcebergSystemFunctionBenchmark.scala | 190 ++++++
.../shuffle/CometCelebornShuffleManagerSuite.scala | 72 ++-
.../CometCelebornShufflePlanningSuite.scala | 37 ++
97 files changed, 9051 insertions(+), 1710 deletions(-)
create mode 100644 native/spark-expr/benches/iceberg_transforms.rs
create mode 100644 native/spark-expr/src/agg_funcs/regr.rs
create mode 100644 native/spark-expr/src/iceberg_funcs/bucket.rs
create mode 100644 native/spark-expr/src/iceberg_funcs/mod.rs
create mode 100644 native/spark-expr/src/iceberg_funcs/temporal.rs
create mode 100644 native/spark-expr/src/iceberg_funcs/truncate.rs
create mode 100644 spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala
create mode 100644 spark/src/test/resources/sql-tests/expressions/array/array_join_eager_eval_dispatch.sql
copy spark/src/test/resources/sql-tests/expressions/array/{array_sort_comparator.sql => array_join_null_replacement.sql} (50%)
create mode 100644 spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala
create mode 100644 spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala
create mode 100644 spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala
create mode 100644 spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala
create mode 100644 spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala
create mode 100644 spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org