You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/05/13 20:12:39 UTC
(datafusion-comet) branch asf-site updated: Publish built docs triggered by 084446291fdda0855858a03888ab96fefd8029e0
This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 63ec42f0e Publish built docs triggered by 084446291fdda0855858a03888ab96fefd8029e0
63ec42f0e is described below
commit 63ec42f0e661c9a864befe764464dba3c9fc046a
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Wed May 13 20:12:35 2026 +0000
Publish built docs triggered by 084446291fdda0855858a03888ab96fefd8029e0
---
_sources/user-guide/0.12/compatibility.md.txt | 197 ---
_sources/user-guide/0.12/expressions.md.txt | 306 ----
_sources/user-guide/0.12/iceberg.md.txt | 151 --
_sources/user-guide/0.12/operators.md.txt | 49 -
_sources/user-guide/0.13/compatibility.md.txt | 2 +-
_sources/user-guide/0.13/configs.md.txt | 2 +-
_sources/user-guide/0.13/datasources.md.txt | 2 +-
_sources/user-guide/0.13/datatypes.md.txt | 2 +-
_sources/user-guide/0.13/expressions.md.txt | 2 +-
_sources/user-guide/0.13/iceberg.md.txt | 2 +-
_sources/user-guide/0.13/installation.md.txt | 2 +-
_sources/user-guide/0.13/kubernetes.md.txt | 2 +-
_sources/user-guide/0.13/metrics.md.txt | 2 +-
_sources/user-guide/0.13/operators.md.txt | 2 +-
_sources/user-guide/0.13/source.md.txt | 2 +-
_sources/user-guide/0.13/tuning.md.txt | 2 +-
_sources/user-guide/0.14/compatibility.md.txt | 2 +-
_sources/user-guide/0.14/configs.md.txt | 2 +-
_sources/user-guide/0.14/datasources.md.txt | 2 +-
_sources/user-guide/0.14/datatypes.md.txt | 2 +-
_sources/user-guide/0.14/expressions.md.txt | 2 +-
_sources/user-guide/0.14/iceberg.md.txt | 2 +-
_sources/user-guide/0.14/installation.md.txt | 2 +-
_sources/user-guide/0.14/kubernetes.md.txt | 2 +-
_sources/user-guide/0.14/metrics.md.txt | 2 +-
_sources/user-guide/0.14/operators.md.txt | 2 +-
_sources/user-guide/0.14/source.md.txt | 2 +-
_sources/user-guide/0.14/tuning.md.txt | 2 +-
_sources/user-guide/0.15/compatibility.md.txt | 3 +
_sources/user-guide/0.15/configs.md.txt | 3 +
_sources/user-guide/0.15/datasources.md.txt | 3 +
_sources/user-guide/0.15/datatypes.md.txt | 3 +
_sources/user-guide/0.15/expressions.md.txt | 3 +
_sources/user-guide/0.15/iceberg.md.txt | 3 +
_sources/user-guide/0.15/installation.md.txt | 3 +
_sources/user-guide/0.15/kubernetes.md.txt | 3 +
_sources/user-guide/0.15/metrics.md.txt | 3 +
_sources/user-guide/0.15/operators.md.txt | 3 +
_sources/user-guide/0.15/source.md.txt | 3 +
_sources/user-guide/0.15/tuning.md.txt | 3 +
.../compatibility/expressions/aggregate.md.txt | 53 +
.../0.16/compatibility/expressions/array.md.txt | 79 +
.../0.16/compatibility/expressions/cast.md.txt | 242 +++
.../0.16/compatibility/expressions/datetime.md.txt | 125 ++
.../compatibility/expressions}/index.md.txt | 32 +-
.../0.16/compatibility/expressions/map.md.txt | 41 +
.../compatibility/expressions/math.md.txt} | 26 +-
.../0.16/compatibility/expressions/misc.md.txt | 53 +
.../0.16/compatibility/expressions/string.md.txt | 118 ++
.../0.16/compatibility/expressions/struct.md.txt | 49 +
.../0.16/compatibility/floating-point.md.txt | 29 +
.../user-guide/0.16/compatibility/index.md.txt | 43 +
.../user-guide/0.16/compatibility/operators.md.txt | 51 +
.../compatibility/regex.md.txt} | 27 +-
.../user-guide/0.16/compatibility/scans.md.txt | 108 ++
.../0.16/compatibility/spark-versions.md.txt | 79 +
_sources/user-guide/{0.12 => 0.16}/configs.md.txt | 87 +-
.../user-guide/{0.12 => 0.16}/datasources.md.txt | 94 +-
.../user-guide/{0.12 => 0.16}/datatypes.md.txt | 5 +-
_sources/user-guide/0.16/expressions.md.txt | 316 ++++
_sources/user-guide/{0.15 => 0.16}/iceberg.md.txt | 20 +-
_sources/user-guide/{0.12 => 0.16}/index.rst.txt | 19 +-
.../user-guide/{0.12 => 0.16}/installation.md.txt | 97 +-
.../user-guide/{0.12 => 0.16}/kubernetes.md.txt | 17 +-
_sources/user-guide/{0.12 => 0.16}/metrics.md.txt | 26 +-
.../user-guide/{0.15 => 0.16}/operators.md.txt | 2 +-
_sources/user-guide/{0.12 => 0.16}/source.md.txt | 24 +-
_sources/user-guide/{0.12 => 0.16}/tuning.md.txt | 49 +-
.../0.16/understanding-comet-plans.md.txt | 232 +++
_sources/user-guide/index.md.txt | 4 +-
contributor-guide/index.html | 4 +-
objects.inv | Bin 1974 -> 2119 bytes
searchindex.js | 2 +-
user-guide/0.12/compatibility.html | 959 -----------
user-guide/0.12/configs.html | 1691 --------------------
user-guide/0.12/datasources.html | 730 ---------
user-guide/0.12/iceberg.html | 644 --------
user-guide/0.12/operators.html | 616 -------
user-guide/0.13/compatibility.html | 6 +-
user-guide/0.13/configs.html | 6 +-
user-guide/0.13/datasources.html | 6 +-
user-guide/0.13/datatypes.html | 6 +-
user-guide/0.13/expressions.html | 6 +-
user-guide/0.13/iceberg.html | 6 +-
user-guide/0.13/index.html | 4 +-
user-guide/0.13/installation.html | 6 +-
user-guide/0.13/kubernetes.html | 12 +-
user-guide/0.13/metrics.html | 6 +-
user-guide/0.13/operators.html | 6 +-
user-guide/0.13/source.html | 6 +-
user-guide/0.13/tuning.html | 6 +-
user-guide/0.14/compatibility.html | 6 +-
user-guide/0.14/configs.html | 6 +-
user-guide/0.14/datasources.html | 6 +-
user-guide/0.14/datatypes.html | 6 +-
user-guide/0.14/expressions.html | 6 +-
user-guide/0.14/iceberg.html | 6 +-
user-guide/0.14/index.html | 4 +-
user-guide/0.14/installation.html | 6 +-
user-guide/0.14/kubernetes.html | 6 +-
user-guide/0.14/metrics.html | 6 +-
user-guide/0.14/operators.html | 6 +-
user-guide/0.14/source.html | 6 +-
user-guide/0.14/tuning.html | 6 +-
user-guide/0.15/compatibility.html | 8 +-
user-guide/0.15/configs.html | 8 +-
user-guide/0.15/datasources.html | 8 +-
user-guide/0.15/datatypes.html | 8 +-
user-guide/0.15/expressions.html | 8 +-
user-guide/0.15/iceberg.html | 8 +-
user-guide/0.15/index.html | 8 +-
user-guide/0.15/installation.html | 8 +-
user-guide/0.15/kubernetes.html | 8 +-
user-guide/0.15/metrics.html | 8 +-
user-guide/0.15/operators.html | 8 +-
user-guide/0.15/source.html | 8 +-
user-guide/0.15/tuning.html | 8 +-
.../compatibility/expressions/aggregate.html | 8 +-
.../compatibility/expressions/array.html | 8 +-
.../compatibility/expressions/cast.html | 29 +-
.../compatibility/expressions/datetime.html | 8 +-
.../compatibility/expressions/index.html | 8 +-
.../compatibility/expressions/map.html | 8 +-
.../compatibility/expressions/math.html | 8 +-
.../compatibility/expressions/misc.html | 8 +-
.../compatibility/expressions/string.html | 8 +-
.../compatibility/expressions/struct.html | 8 +-
.../compatibility/floating-point.html | 8 +-
.../{latest => 0.16}/compatibility/index.html | 8 +-
.../{latest => 0.16}/compatibility/operators.html | 8 +-
.../{latest => 0.16}/compatibility/regex.html | 8 +-
.../{latest => 0.16}/compatibility/scans.html | 8 +-
.../compatibility/spark-versions.html | 16 +-
user-guide/{latest => 0.16}/configs.html | 82 +-
user-guide/{latest => 0.16}/datasources.html | 10 +-
user-guide/{0.12 => 0.16}/datatypes.html | 23 +-
user-guide/{0.12 => 0.16}/expressions.html | 507 ++----
user-guide/{latest => 0.16}/iceberg.html | 14 +-
user-guide/{0.12 => 0.16}/index.html | 51 +-
user-guide/{0.12 => 0.16}/installation.html | 136 +-
user-guide/{0.12 => 0.16}/kubernetes.html | 45 +-
user-guide/{0.12 => 0.16}/metrics.html | 45 +-
user-guide/{latest => 0.16}/operators.html | 8 +-
user-guide/{0.12 => 0.16}/source.html | 39 +-
user-guide/{0.12 => 0.16}/tuning.html | 66 +-
.../understanding-comet-plans.html | 8 +-
user-guide/index.html | 4 +-
.../compatibility/expressions/aggregate.html | 4 +-
.../latest/compatibility/expressions/array.html | 4 +-
.../latest/compatibility/expressions/cast.html | 4 +-
.../latest/compatibility/expressions/datetime.html | 4 +-
.../latest/compatibility/expressions/index.html | 4 +-
.../latest/compatibility/expressions/map.html | 4 +-
.../latest/compatibility/expressions/math.html | 4 +-
.../latest/compatibility/expressions/misc.html | 4 +-
.../latest/compatibility/expressions/string.html | 4 +-
.../latest/compatibility/expressions/struct.html | 4 +-
.../latest/compatibility/floating-point.html | 4 +-
user-guide/latest/compatibility/index.html | 4 +-
user-guide/latest/compatibility/operators.html | 4 +-
user-guide/latest/compatibility/regex.html | 4 +-
user-guide/latest/compatibility/scans.html | 4 +-
.../latest/compatibility/spark-versions.html | 4 +-
user-guide/latest/configs.html | 4 +-
user-guide/latest/datasources.html | 4 +-
user-guide/latest/datatypes.html | 4 +-
user-guide/latest/expressions.html | 4 +-
user-guide/latest/iceberg.html | 4 +-
user-guide/latest/index.html | 4 +-
user-guide/latest/installation.html | 4 +-
user-guide/latest/kubernetes.html | 10 +-
user-guide/latest/metrics.html | 4 +-
user-guide/latest/operators.html | 4 +-
user-guide/latest/source.html | 4 +-
user-guide/latest/tuning.html | 4 +-
user-guide/latest/understanding-comet-plans.html | 4 +-
176 files changed, 2680 insertions(+), 6479 deletions(-)
diff --git a/_sources/user-guide/0.12/compatibility.md.txt b/_sources/user-guide/0.12/compatibility.md.txt
deleted file mode 100644
index d98bcbe7d..000000000
--- a/_sources/user-guide/0.12/compatibility.md.txt
+++ /dev/null
@@ -1,197 +0,0 @@
-<!---
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
-
-# Compatibility Guide
-
-Comet aims to provide consistent results with the version of Apache Spark that is being used.
-
-This guide offers information about areas of functionality where there are known differences.
-
-## Parquet
-
-Comet has the following limitations when reading Parquet files:
-
-- Comet does not support reading decimals encoded in binary format.
-- No support for default values that are nested types (e.g., maps, arrays, structs). Literal default values are supported.
-
-## ANSI Mode
-
-Comet will fall back to Spark for the following expressions when ANSI mode is enabled. Thes expressions can be enabled by setting
-`spark.comet.expression.EXPRNAME.allowIncompatible=true`, where `EXPRNAME` is the Spark expression class name. See
-the [Comet Supported Expressions Guide](expressions.md) for more information on this configuration setting.
-
-- Average
-- Sum
-- Cast (in some cases)
-
-There is an [epic](https://github.com/apache/datafusion-comet/issues/313) where we are tracking the work to fully implement ANSI support.
-
-## Floating-point Number Comparison
-
-Spark normalizes NaN and zero for floating point numbers for several cases. See `NormalizeFloatingNumbers` optimization rule in Spark.
-However, one exception is comparison. Spark does not normalize NaN and zero when comparing values
-because they are handled well in Spark (e.g., `SQLOrderingUtil.compareFloats`). But the comparison
-functions of arrow-rs used by DataFusion do not normalize NaN and zero (e.g., [arrow::compute::kernels::cmp::eq](https://docs.rs/arrow/latest/arrow/compute/kernels/cmp/fn.eq.html#)).
-So Comet adds additional normalization expression of NaN and zero for comparisons, and may still have differences
-to Spark in some cases, especially when the data contains both positive and negative zero. This is likely an edge
-case that is not of concern for many users. If it is a concern, setting `spark.comet.exec.strictFloatingPoint=true`
-will make relevant operations fall back to Spark.
-
-## Incompatible Expressions
-
-Expressions that are not 100% Spark-compatible will fall back to Spark by default and can be enabled by setting
-`spark.comet.expression.EXPRNAME.allowIncompatible=true`, where `EXPRNAME` is the Spark expression class name. See
-the [Comet Supported Expressions Guide](expressions.md) for more information on this configuration setting.
-
-## Regular Expressions
-
-Comet uses the Rust regexp crate for evaluating regular expressions, and this has different behavior from Java's
-regular expression engine. Comet will fall back to Spark for patterns that are known to produce different results, but
-this can be overridden by setting `spark.comet.regexp.allowIncompatible=true`.
-
-## Window Functions
-
-Comet's support for window functions is incomplete and known to be incorrect. It is disabled by default and
-should not be used in production. The feature will be enabled in a future release. Tracking issue: [#2721](https://github.com/apache/datafusion-comet/issues/2721).
-
-## Cast
-
-Cast operations in Comet fall into three levels of support:
-
-- **Compatible**: The results match Apache Spark
-- **Incompatible**: The results may match Apache Spark for some inputs, but there are known issues where some inputs
- will result in incorrect results or exceptions. The query stage will fall back to Spark by default. Setting
- `spark.comet.expression.Cast.allowIncompatible=true` will allow all incompatible casts to run natively in Comet, but this is not
- recommended for production use.
-- **Unsupported**: Comet does not provide a native version of this cast expression and the query stage will fall back to
- Spark.
-
-### Compatible Casts
-
-The following cast operations are generally compatible with Spark except for the differences noted here.
-
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-
-<!--BEGIN:COMPAT_CAST_TABLE-->
-<!-- prettier-ignore-start -->
-| From Type | To Type | Notes |
-|-|-|-|
-| boolean | byte | |
-| boolean | short | |
-| boolean | integer | |
-| boolean | long | |
-| boolean | float | |
-| boolean | double | |
-| boolean | string | |
-| byte | boolean | |
-| byte | short | |
-| byte | integer | |
-| byte | long | |
-| byte | float | |
-| byte | double | |
-| byte | decimal | |
-| byte | string | |
-| short | boolean | |
-| short | byte | |
-| short | integer | |
-| short | long | |
-| short | float | |
-| short | double | |
-| short | decimal | |
-| short | string | |
-| integer | boolean | |
-| integer | byte | |
-| integer | short | |
-| integer | long | |
-| integer | float | |
-| integer | double | |
-| integer | decimal | |
-| integer | string | |
-| long | boolean | |
-| long | byte | |
-| long | short | |
-| long | integer | |
-| long | float | |
-| long | double | |
-| long | decimal | |
-| long | string | |
-| float | boolean | |
-| float | byte | |
-| float | short | |
-| float | integer | |
-| float | long | |
-| float | double | |
-| float | string | There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45 |
-| double | boolean | |
-| double | byte | |
-| double | short | |
-| double | integer | |
-| double | long | |
-| double | float | |
-| double | string | There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45 |
-| decimal | boolean | |
-| decimal | byte | |
-| decimal | short | |
-| decimal | integer | |
-| decimal | long | |
-| decimal | float | |
-| decimal | double | |
-| decimal | decimal | |
-| decimal | string | There can be formatting differences in some case due to Spark using scientific notation where Comet does not |
-| string | boolean | |
-| string | byte | |
-| string | short | |
-| string | integer | |
-| string | long | |
-| string | binary | |
-| string | date | Only supports years between 262143 BC and 262142 AD |
-| binary | string | |
-| date | string | |
-| timestamp | long | |
-| timestamp | string | |
-| timestamp | date | |
-<!-- prettier-ignore-end -->
-<!--END:COMPAT_CAST_TABLE-->
-
-### Incompatible Casts
-
-The following cast operations are not compatible with Spark for all inputs and are disabled by default.
-
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-
-<!--BEGIN:INCOMPAT_CAST_TABLE-->
-<!-- prettier-ignore-start -->
-| From Type | To Type | Notes |
-|-|-|-|
-| float | decimal | There can be rounding differences |
-| double | decimal | There can be rounding differences |
-| string | float | Does not support inputs ending with 'd' or 'f'. Does not support 'inf'. Does not support ANSI mode. |
-| string | double | Does not support inputs ending with 'd' or 'f'. Does not support 'inf'. Does not support ANSI mode. |
-| string | decimal | Does not support inputs ending with 'd' or 'f'. Does not support 'inf'. Does not support ANSI mode. Returns 0.0 instead of null if input contains no digits |
-| string | timestamp | Not all valid formats are supported |
-<!-- prettier-ignore-end -->
-<!--END:INCOMPAT_CAST_TABLE-->
-
-### Unsupported Casts
-
-Any cast not listed in the previous tables is currently unsupported. We are working on adding more. See the
-[tracking issue](https://github.com/apache/datafusion-comet/issues/286) for more details.
diff --git a/_sources/user-guide/0.12/expressions.md.txt b/_sources/user-guide/0.12/expressions.md.txt
deleted file mode 100644
index 0f75d001f..000000000
--- a/_sources/user-guide/0.12/expressions.md.txt
+++ /dev/null
@@ -1,306 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
-
-# Supported Spark Expressions
-
-Comet supports the following Spark expressions. Expressions that are marked as Spark-compatible will either run
-natively in Comet and provide the same results as Spark, or will fall back to Spark for cases that would not
-be compatible.
-
-All expressions are enabled by default, but most can be disabled by setting
-`spark.comet.expression.EXPRNAME.enabled=false`, where `EXPRNAME` is the expression name as specified in
-the following tables, such as `Length`, or `StartsWith`. See the [Comet Configuration Guide] for a full list
-of expressions that be disabled.
-
-Expressions that are not Spark-compatible will fall back to Spark by default and can be enabled by setting
-`spark.comet.expression.EXPRNAME.allowIncompatible=true`.
-
-## Conditional Expressions
-
-| Expression | SQL | Spark-Compatible? |
-| ---------- | ------------------------------------------- | ----------------- |
-| CaseWhen | `CASE WHEN expr THEN expr ELSE expr END` | Yes |
-| If | `IF(predicate_expr, true_expr, false_expr)` | Yes |
-
-## Predicate Expressions
-
-| Expression | SQL | Spark-Compatible? |
-| ------------------ | ------------- | ----------------- |
-| And | `AND` | Yes |
-| EqualTo | `=` | Yes |
-| EqualNullSafe | `<=>` | Yes |
-| GreaterThan | `>` | Yes |
-| GreaterThanOrEqual | `>=` | Yes |
-| LessThan | `<` | Yes |
-| LessThanOrEqual | `<=` | Yes |
-| In | `IN` | Yes |
-| IsNotNull | `IS NOT NULL` | Yes |
-| IsNull | `IS NULL` | Yes |
-| InSet | `IN (...)` | Yes |
-| Not | `NOT` | Yes |
-| Or | `OR` | Yes |
-
-## String Functions
-
-| Expression | Spark-Compatible? | Compatibility Notes |
-| --------------- | ----------------- | ---------------------------------------------------------------------------------------------------------- |
-| Ascii | Yes | |
-| BitLength | Yes | |
-| Chr | Yes | |
-| Concat | Yes | Only string inputs are supported |
-| ConcatWs | Yes | |
-| Contains | Yes | |
-| EndsWith | Yes | |
-| InitCap | No | Behavior is different in some cases, such as hyphenated names. |
-| Length | Yes | |
-| Like | Yes | |
-| Lower | No | Results can vary depending on locale and character set. Requires `spark.comet.caseConversion.enabled=true` |
-| OctetLength | Yes | |
-| Reverse | Yes | |
-| RLike | No | Uses Rust regexp engine, which has different behavior to Java regexp engine |
-| StartsWith | Yes | |
-| StringInstr | Yes | |
-| StringRepeat | Yes | Negative argument for number of times to repeat causes exception |
-| StringReplace | Yes | |
-| StringLPad | Yes | |
-| StringRPad | Yes | |
-| StringSpace | Yes | |
-| StringTranslate | Yes | |
-| StringTrim | Yes | |
-| StringTrimBoth | Yes | |
-| StringTrimLeft | Yes | |
-| StringTrimRight | Yes | |
-| Substring | Yes | |
-| Upper | No | Results can vary depending on locale and character set. Requires `spark.comet.caseConversion.enabled=true` |
-
-## Date/Time Functions
-
-| Expression | SQL | Spark-Compatible? | Compatibility Notes |
-| -------------- | ---------------------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------- |
-| DateAdd | `date_add` | Yes | |
-| DateSub | `date_sub` | Yes | |
-| DatePart | `date_part(field, source)` | Yes | Supported values of `field`: `year`/`month`/`week`/`day`/`dayofweek`/`dayofweek_iso`/`doy`/`quarter`/`hour`/`minute` |
-| Extract | `extract(field FROM source)` | Yes | Supported values of `field`: `year`/`month`/`week`/`day`/`dayofweek`/`dayofweek_iso`/`doy`/`quarter`/`hour`/`minute` |
-| FromUnixTime | `from_unixtime` | No | Does not support format, supports only -8334601211038 <= sec <= 8210266876799 |
-| Hour | `hour` | Yes | |
-| Minute | `minute` | Yes | |
-| Second | `second` | Yes | |
-| TruncDate | `trunc` | Yes | |
-| TruncTimestamp | `trunc_date` | Yes | |
-| Year | `year` | Yes | |
-| Month | `month` | Yes | |
-| DayOfMonth | `day`/`dayofmonth` | Yes | |
-| DayOfWeek | `dayofweek` | Yes | |
-| WeekDay | `weekday` | Yes | |
-| DayOfYear | `dayofyear` | Yes | |
-| WeekOfYear | `weekofyear` | Yes | |
-| Quarter | `quarter` | Yes | |
-
-## Math Expressions
-
-| Expression | SQL | Spark-Compatible? | Compatibility Notes |
-| -------------- | --------- | ----------------- | --------------------------------- |
-| Abs | `abs` | Yes | |
-| Acos | `acos` | Yes | |
-| Add | `+` | Yes | |
-| Asin | `asin` | Yes | |
-| Atan | `atan` | Yes | |
-| Atan2 | `atan2` | Yes | |
-| BRound | `bround` | Yes | |
-| Ceil | `ceil` | Yes | |
-| Cos | `cos` | Yes | |
-| Cosh | `cosh` | Yes | |
-| Cot | `cot` | Yes | |
-| Divide | `/` | Yes | |
-| Exp | `exp` | Yes | |
-| Expm1 | `expm1` | Yes | |
-| Floor | `floor` | Yes | |
-| Hex | `hex` | Yes | |
-| IntegralDivide | `div` | Yes | |
-| IsNaN | `isnan` | Yes | |
-| Log | `log` | Yes | |
-| Log2 | `log2` | Yes | |
-| Log10 | `log10` | Yes | |
-| Multiply | `*` | Yes | |
-| Pow | `power` | Yes | |
-| Rand | `rand` | Yes | |
-| Randn | `randn` | Yes | |
-| Remainder | `%` | Yes | |
-| Round | `round` | Yes | |
-| Signum | `signum` | Yes | |
-| Sin | `sin` | Yes | |
-| Sinh | `sinh` | Yes | |
-| Sqrt | `sqrt` | Yes | |
-| Subtract | `-` | Yes | |
-| Tan | `tan` | Yes | |
-| Tanh | `tanh` | Yes | |
-| TryAdd | `try_add` | Yes | Only integer inputs are supported |
-| TryDivide | `try_div` | Yes | Only integer inputs are supported |
-| TryMultiply | `try_mul` | Yes | Only integer inputs are supported |
-| TrySubtract | `try_sub` | Yes | Only integer inputs are supported |
-| UnaryMinus | `-` | Yes | |
-| Unhex | `unhex` | Yes | |
-
-## Hashing Functions
-
-| Expression | Spark-Compatible? |
-| ----------- | ----------------- |
-| Md5 | Yes |
-| Murmur3Hash | Yes |
-| Sha2 | Yes |
-| XxHash64 | Yes |
-
-## Bitwise Expressions
-
-| Expression | SQL | Spark-Compatible? |
-| ------------ | ---- | ----------------- |
-| BitwiseAnd | `&` | Yes |
-| BitwiseCount | | Yes |
-| BitwiseGet | | Yes |
-| BitwiseOr | `\|` | Yes |
-| BitwiseNot | `~` | Yes |
-| BitwiseXor | `^` | Yes |
-| ShiftLeft | `<<` | Yes |
-| ShiftRight | `>>` | Yes |
-
-## Aggregate Expressions
-
-| Expression | SQL | Spark-Compatible? | Compatibility Notes |
-| ------------- | ---------- | ------------------------- | ---------------------------------------------------------------- |
-| Average | | Yes, except for ANSI mode | |
-| BitAndAgg | | Yes | |
-| BitOrAgg | | Yes | |
-| BitXorAgg | | Yes | |
-| BoolAnd | `bool_and` | Yes | |
-| BoolOr | `bool_or` | Yes | |
-| Corr | | Yes | |
-| Count | | Yes | |
-| CovPopulation | | Yes | |
-| CovSample | | Yes | |
-| First | | No | This function is not deterministic. Results may not match Spark. |
-| Last | | No | This function is not deterministic. Results may not match Spark. |
-| Max | | Yes | |
-| Min | | Yes | |
-| StddevPop | | Yes | |
-| StddevSamp | | Yes | |
-| Sum | | Yes, except for ANSI mode | |
-| VariancePop | | Yes | |
-| VarianceSamp | | Yes | |
-
-## Window Functions
-
-```{warning}
-Window support is disabled by default due to known correctness issues. Tracking issue: [#2721](https://github.com/apache/datafusion-comet/issues/2721).
-```
-
-Comet supports using the following aggregate functions within window contexts with PARTITION BY and ORDER BY clauses.
-
-| Expression | Spark-Compatible? | Compatibility Notes |
-| ---------- | ----------------- | ------------------- |
-| Count | Yes | |
-| Max | Yes | |
-| Min | Yes | |
-| Sum | Yes | |
-
-**Note:** Dedicated window functions such as `rank`, `dense_rank`, `row_number`, `lag`, `lead`, `ntile`, `cume_dist`, `percent_rank`, and `nth_value` are not currently supported and will fall back to Spark.
-
-## Array Expressions
-
-| Expression | Spark-Compatible? | Compatibility Notes |
-| -------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| ArrayAppend | No | |
-| ArrayCompact | No | |
-| ArrayContains | Yes | |
-| ArrayDistinct | No | Behaves differently than spark. Comet first sorts then removes duplicates while Spark preserves the original order. |
-| ArrayExcept | No | |
-| ArrayFilter | Yes | Only supports case where function is `IsNotNull` |
-| ArrayInsert | No | |
-| ArrayIntersect | No | |
-| ArrayJoin | No | |
-| ArrayMax | Yes | |
-| ArrayMin | Yes | |
-| ArrayRemove | Yes | |
-| ArrayRepeat | No | |
-| ArrayUnion | No | Behaves differently than spark. Comet sorts the input arrays before performing the union, while Spark preserves the order of the first array and appends unique elements from the second. |
-| ArraysOverlap | No | |
-| CreateArray | Yes | |
-| ElementAt | Yes | Input must be an array. Map inputs are not supported. |
-| Flatten | Yes | |
-| GetArrayItem | Yes | |
-
-## Map Expressions
-
-| Expression | Spark-Compatible? |
-| ------------- | ----------------- |
-| GetMapValue | Yes |
-| MapKeys | Yes |
-| MapEntries | Yes |
-| MapValues | Yes |
-| MapFromArrays | Yes |
-
-## Struct Expressions
-
-| Expression | Spark-Compatible? |
-| -------------------- | ----------------- |
-| CreateNamedStruct | Yes |
-| GetArrayStructFields | Yes |
-| GetStructField | Yes |
-| StructsToJson | Yes |
-
-## Conversion Expressions
-
-| Expression | Spark-Compatible | Compatibility Notes |
-| ---------- | ------------------------ | ------------------------------------------------------------------------------------------- |
-| Cast | Depends on specific cast | See the [Comet Compatibility Guide] for list of supported cast expressions and known issues |
-
-## SortOrder
-
-| Expression | Spark-Compatible? | Compatibility Notes |
-| ---------- | ----------------- | ------------------- |
-| NullsFirst | Yes | |
-| NullsLast | Yes | |
-| Ascending | Yes | |
-| Descending | Yes | |
-
-## Other
-
-| Expression | Spark-Compatible? | Compatibility Notes |
-| ---------------------------- | ----------------- | --------------------------------------------------------------------------- |
-| Alias | Yes | |
-| AttributeReference | Yes | |
-| BloomFilterMightContain | Yes | |
-| Coalesce | Yes | |
-| CheckOverflow | Yes | |
-| KnownFloatingPointNormalized | Yes | |
-| Literal | Yes | |
-| MakeDecimal | Yes | |
-| MonotonicallyIncreasingID | Yes | |
-| NormalizeNaNAndZero | Yes | |
-| PromotePrecision | Yes | |
-| RegExpReplace | No | Uses Rust regexp engine, which has different behavior to Java regexp engine |
-| ScalarSubquery | Yes | |
-| SparkPartitionID | Yes | |
-| ToPrettyString | Yes | |
-| UnscaledValue | Yes | |
-
-[Comet Configuration Guide]: configs.md
-[Comet Compatibility Guide]: compatibility.md
diff --git a/_sources/user-guide/0.12/iceberg.md.txt b/_sources/user-guide/0.12/iceberg.md.txt
deleted file mode 100644
index d45fe4232..000000000
--- a/_sources/user-guide/0.12/iceberg.md.txt
+++ /dev/null
@@ -1,151 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
-
-# Accelerating Apache Iceberg Parquet Scans using Comet (Experimental)
-
-**Note: Iceberg integration is a work-in-progress. It is currently necessary to build Iceberg from
-source rather than using available artifacts in Maven**
-
-## Build Comet
-
-Run a Maven install so that we can compile Iceberg against latest Comet:
-
-```shell
-mvn install -DskipTests
-```
-
-Build the release JAR to be used from Spark:
-
-```shell
-make release
-```
-
-Set `COMET_JAR` env var:
-
-```shell
-export COMET_JAR=`pwd`/spark/target/comet-spark-spark3.5_2.12-0.12.0.jar
-```
-
-## Build Iceberg
-
-Clone the Iceberg repository and apply code changes needed by Comet
-
-```shell
-git clone git@github.com:apache/iceberg.git
-cd iceberg
-git checkout apache-iceberg-1.8.1
-git apply ../datafusion-comet/dev/diffs/iceberg/1.8.1.diff
-```
-
-Perform a clean build
-
-```shell
-./gradlew clean
-./gradlew build -x test -x integrationTest
-```
-
-## Test
-
-Set `ICEBERG_JAR` environment variable.
-
-```shell
-export ICEBERG_JAR=`pwd`/spark/v3.5/spark-runtime/build/libs/iceberg-spark-runtime-3.5_2.12-1.9.0-SNAPSHOT.jar
-```
-
-Launch Spark Shell:
-
-```shell
-$SPARK_HOME/bin/spark-shell \
- --jars $COMET_JAR,$ICEBERG_JAR \
- --conf spark.driver.extraClassPath=$COMET_JAR,$ICEBERG_JAR \
- --conf spark.executor.extraClassPath=$COMET_JAR,$ICEBERG_JAR \
- --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
- --conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkCatalog \
- --conf spark.sql.catalog.spark_catalog.type=hadoop \
- --conf spark.sql.catalog.spark_catalog.warehouse=/tmp/warehouse \
- --conf spark.plugins=org.apache.spark.CometPlugin \
- --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
- --conf spark.sql.iceberg.parquet.reader-type=COMET \
- --conf spark.comet.explainFallback.enabled=true \
- --conf spark.memory.offHeap.enabled=true \
- --conf spark.memory.offHeap.size=2g \
- --conf spark.comet.use.lazyMaterialization=false \
- --conf spark.comet.schemaEvolution.enabled=true
-```
-
-Create an Iceberg table. Note that Comet will not accelerate this part.
-
-```
-scala> spark.sql(s"CREATE TABLE IF NOT EXISTS t1 (c0 INT, c1 STRING) USING iceberg")
-scala> spark.sql(s"INSERT INTO t1 VALUES ${(0 until 10000).map(i => (i, i)).mkString(",")}")
-```
-
-Comet should now be able to accelerate reading the table:
-
-```
-scala> spark.sql(s"SELECT * from t1").show()
-```
-
-This should produce the following output:
-
-```
-scala> spark.sql(s"SELECT * from t1").show()
-+---+---+
-| c0| c1|
-+---+---+
-| 0| 0|
-| 1| 1|
-| 2| 2|
-| 3| 3|
-| 4| 4|
-| 5| 5|
-| 6| 6|
-| 7| 7|
-| 8| 8|
-| 9| 9|
-| 10| 10|
-| 11| 11|
-| 12| 12|
-| 13| 13|
-| 14| 14|
-| 15| 15|
-| 16| 16|
-| 17| 17|
-| 18| 18|
-| 19| 19|
-+---+---+
-only showing top 20 rows
-```
-
-Confirm that the query was accelerated by Comet:
-
-```
-scala> spark.sql(s"SELECT * from t1").explain()
-== Physical Plan ==
-*(1) CometColumnarToRow
-+- CometBatchScan spark_catalog.default.t1[c0#26, c1#27] spark_catalog.default.t1 (branch=null) [filters=, groupedBy=] RuntimeFilters: []
-```
-
-## Known issues
-
-- Spark Runtime Filtering isn't [working](https://github.com/apache/datafusion-comet/issues/2116)
- - You can bypass the issue by either setting `spark.sql.adaptive.enabled=false` or `spark.comet.exec.broadcastExchange.enabled=false`
diff --git a/_sources/user-guide/0.12/operators.md.txt b/_sources/user-guide/0.12/operators.md.txt
deleted file mode 100644
index bbe58c1e0..000000000
--- a/_sources/user-guide/0.12/operators.md.txt
+++ /dev/null
@@ -1,49 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
-
-# Supported Spark Operators
-
-The following Spark operators are currently replaced with native versions. Query stages that contain any operators
-not supported by Comet will fall back to regular Spark execution.
-
-| Operator | Spark-Compatible? | Compatibility Notes |
-| ----------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------ |
-| BatchScanExec | Yes | Supports Parquet files and Apache Iceberg Parquet scans. See the [Comet Compatibility Guide] for more information. |
-| BroadcastExchangeExec | Yes | |
-| BroadcastHashJoinExec | Yes | |
-| ExpandExec | Yes | |
-| FileSourceScanExec | Yes | Supports Parquet files. See the [Comet Compatibility Guide] for more information. |
-| FilterExec | Yes | |
-| GlobalLimitExec | Yes | |
-| HashAggregateExec | Yes | |
-| LocalLimitExec | Yes | |
-| LocalTableScanExec | No | Experimental and disabled by default. |
-| ObjectHashAggregateExec | Yes | Supports a limited number of aggregates, such as `bloom_filter_agg`. |
-| ProjectExec | Yes | |
-| ShuffleExchangeExec | Yes | |
-| ShuffledHashJoinExec | Yes | |
-| SortExec | Yes | |
-| SortMergeJoinExec | Yes | |
-| UnionExec | Yes | |
-| WindowExec | No | Disabled by default due to known correctness issues. |
-
-[Comet Compatibility Guide]: compatibility.md
diff --git a/_sources/user-guide/0.13/compatibility.md.txt b/_sources/user-guide/0.13/compatibility.md.txt
index dc4624b98..00ea5a4fa 100644
--- a/_sources/user-guide/0.13/compatibility.md.txt
+++ b/_sources/user-guide/0.13/compatibility.md.txt
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Compatibility Guide
diff --git a/_sources/user-guide/0.13/configs.md.txt b/_sources/user-guide/0.13/configs.md.txt
index 7cb2e9c88..43cf7d13a 100644
--- a/_sources/user-guide/0.13/configs.md.txt
+++ b/_sources/user-guide/0.13/configs.md.txt
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Comet Configuration Settings
diff --git a/_sources/user-guide/0.13/datasources.md.txt b/_sources/user-guide/0.13/datasources.md.txt
index b074f82fc..bd937d85b 100644
--- a/_sources/user-guide/0.13/datasources.md.txt
+++ b/_sources/user-guide/0.13/datasources.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Supported Spark Data Sources
diff --git a/_sources/user-guide/0.13/datatypes.md.txt b/_sources/user-guide/0.13/datatypes.md.txt
index e68dd9e84..4c9e56612 100644
--- a/_sources/user-guide/0.13/datatypes.md.txt
+++ b/_sources/user-guide/0.13/datatypes.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Supported Spark Data Types
diff --git a/_sources/user-guide/0.13/expressions.md.txt b/_sources/user-guide/0.13/expressions.md.txt
index d7e459813..8f1b6d13c 100644
--- a/_sources/user-guide/0.13/expressions.md.txt
+++ b/_sources/user-guide/0.13/expressions.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Supported Spark Expressions
diff --git a/_sources/user-guide/0.13/iceberg.md.txt b/_sources/user-guide/0.13/iceberg.md.txt
index 88e156116..32a1171d6 100644
--- a/_sources/user-guide/0.13/iceberg.md.txt
+++ b/_sources/user-guide/0.13/iceberg.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Accelerating Apache Iceberg Parquet Scans using Comet (Experimental)
diff --git a/_sources/user-guide/0.13/installation.md.txt b/_sources/user-guide/0.13/installation.md.txt
index bbb253e67..4f2332221 100644
--- a/_sources/user-guide/0.13/installation.md.txt
+++ b/_sources/user-guide/0.13/installation.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Installing DataFusion Comet
diff --git a/_sources/user-guide/0.13/kubernetes.md.txt b/_sources/user-guide/0.13/kubernetes.md.txt
index c86d983f5..864881dfe 100644
--- a/_sources/user-guide/0.13/kubernetes.md.txt
+++ b/_sources/user-guide/0.13/kubernetes.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Comet Kubernetes Support
diff --git a/_sources/user-guide/0.13/metrics.md.txt b/_sources/user-guide/0.13/metrics.md.txt
index 83169dfa2..7caa948e6 100644
--- a/_sources/user-guide/0.13/metrics.md.txt
+++ b/_sources/user-guide/0.13/metrics.md.txt
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Comet Metrics
diff --git a/_sources/user-guide/0.13/operators.md.txt b/_sources/user-guide/0.13/operators.md.txt
index bc703d2ee..5b41c8879 100644
--- a/_sources/user-guide/0.13/operators.md.txt
+++ b/_sources/user-guide/0.13/operators.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Supported Spark Operators
diff --git a/_sources/user-guide/0.13/source.md.txt b/_sources/user-guide/0.13/source.md.txt
index 77d30a003..6e0b6381a 100644
--- a/_sources/user-guide/0.13/source.md.txt
+++ b/_sources/user-guide/0.13/source.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Building Comet From Source
diff --git a/_sources/user-guide/0.13/tuning.md.txt b/_sources/user-guide/0.13/tuning.md.txt
index 3c4a66c21..fcfa72006 100644
--- a/_sources/user-guide/0.13/tuning.md.txt
+++ b/_sources/user-guide/0.13/tuning.md.txt
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Comet Tuning Guide
diff --git a/_sources/user-guide/0.14/compatibility.md.txt b/_sources/user-guide/0.14/compatibility.md.txt
index efa1b8776..8325307a3 100644
--- a/_sources/user-guide/0.14/compatibility.md.txt
+++ b/_sources/user-guide/0.14/compatibility.md.txt
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Compatibility Guide
diff --git a/_sources/user-guide/0.14/configs.md.txt b/_sources/user-guide/0.14/configs.md.txt
index c6e514f33..6ae6a0f80 100644
--- a/_sources/user-guide/0.14/configs.md.txt
+++ b/_sources/user-guide/0.14/configs.md.txt
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Comet Configuration Settings
diff --git a/_sources/user-guide/0.14/datasources.md.txt b/_sources/user-guide/0.14/datasources.md.txt
index 331f97bc6..3e53ab71d 100644
--- a/_sources/user-guide/0.14/datasources.md.txt
+++ b/_sources/user-guide/0.14/datasources.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Supported Spark Data Sources
diff --git a/_sources/user-guide/0.14/datatypes.md.txt b/_sources/user-guide/0.14/datatypes.md.txt
index e68dd9e84..4c9e56612 100644
--- a/_sources/user-guide/0.14/datatypes.md.txt
+++ b/_sources/user-guide/0.14/datatypes.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Supported Spark Data Types
diff --git a/_sources/user-guide/0.14/expressions.md.txt b/_sources/user-guide/0.14/expressions.md.txt
index 7f0d51d98..63a842087 100644
--- a/_sources/user-guide/0.14/expressions.md.txt
+++ b/_sources/user-guide/0.14/expressions.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Supported Spark Expressions
diff --git a/_sources/user-guide/0.14/iceberg.md.txt b/_sources/user-guide/0.14/iceberg.md.txt
index 3381b90ef..cc66cc3c5 100644
--- a/_sources/user-guide/0.14/iceberg.md.txt
+++ b/_sources/user-guide/0.14/iceberg.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Accelerating Apache Iceberg Parquet Scans using Comet (Experimental)
diff --git a/_sources/user-guide/0.14/installation.md.txt b/_sources/user-guide/0.14/installation.md.txt
index a5aa555ca..b530c5f93 100644
--- a/_sources/user-guide/0.14/installation.md.txt
+++ b/_sources/user-guide/0.14/installation.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Installing DataFusion Comet
diff --git a/_sources/user-guide/0.14/kubernetes.md.txt b/_sources/user-guide/0.14/kubernetes.md.txt
index c86d983f5..864881dfe 100644
--- a/_sources/user-guide/0.14/kubernetes.md.txt
+++ b/_sources/user-guide/0.14/kubernetes.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Comet Kubernetes Support
diff --git a/_sources/user-guide/0.14/metrics.md.txt b/_sources/user-guide/0.14/metrics.md.txt
index 83169dfa2..7caa948e6 100644
--- a/_sources/user-guide/0.14/metrics.md.txt
+++ b/_sources/user-guide/0.14/metrics.md.txt
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Comet Metrics
diff --git a/_sources/user-guide/0.14/operators.md.txt b/_sources/user-guide/0.14/operators.md.txt
index bc703d2ee..5b41c8879 100644
--- a/_sources/user-guide/0.14/operators.md.txt
+++ b/_sources/user-guide/0.14/operators.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Supported Spark Operators
diff --git a/_sources/user-guide/0.14/source.md.txt b/_sources/user-guide/0.14/source.md.txt
index 142c4eca7..26bcd9938 100644
--- a/_sources/user-guide/0.14/source.md.txt
+++ b/_sources/user-guide/0.14/source.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Building Comet From Source
diff --git a/_sources/user-guide/0.14/tuning.md.txt b/_sources/user-guide/0.14/tuning.md.txt
index bfdaaa640..db219ab56 100644
--- a/_sources/user-guide/0.14/tuning.md.txt
+++ b/_sources/user-guide/0.14/tuning.md.txt
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
```
# Comet Tuning Guide
diff --git a/_sources/user-guide/0.15/compatibility.md.txt b/_sources/user-guide/0.15/compatibility.md.txt
index 4539e8c08..c5f2e49ae 100644
--- a/_sources/user-guide/0.15/compatibility.md.txt
+++ b/_sources/user-guide/0.15/compatibility.md.txt
@@ -16,6 +16,9 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Compatibility Guide
diff --git a/_sources/user-guide/0.15/configs.md.txt b/_sources/user-guide/0.15/configs.md.txt
index 664d3769c..ad1db3481 100644
--- a/_sources/user-guide/0.15/configs.md.txt
+++ b/_sources/user-guide/0.15/configs.md.txt
@@ -16,6 +16,9 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Comet Configuration Settings
diff --git a/_sources/user-guide/0.15/datasources.md.txt b/_sources/user-guide/0.15/datasources.md.txt
index c7c596e00..ff6e61196 100644
--- a/_sources/user-guide/0.15/datasources.md.txt
+++ b/_sources/user-guide/0.15/datasources.md.txt
@@ -16,6 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Supported Spark Data Sources
diff --git a/_sources/user-guide/0.15/datatypes.md.txt b/_sources/user-guide/0.15/datatypes.md.txt
index 6d7939348..4c9e56612 100644
--- a/_sources/user-guide/0.15/datatypes.md.txt
+++ b/_sources/user-guide/0.15/datatypes.md.txt
@@ -16,6 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Supported Spark Data Types
diff --git a/_sources/user-guide/0.15/expressions.md.txt b/_sources/user-guide/0.15/expressions.md.txt
index c4ab53181..5253bff49 100644
--- a/_sources/user-guide/0.15/expressions.md.txt
+++ b/_sources/user-guide/0.15/expressions.md.txt
@@ -16,6 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Supported Spark Expressions
diff --git a/_sources/user-guide/0.15/iceberg.md.txt b/_sources/user-guide/0.15/iceberg.md.txt
index 4dbd085c2..9bdee8c00 100644
--- a/_sources/user-guide/0.15/iceberg.md.txt
+++ b/_sources/user-guide/0.15/iceberg.md.txt
@@ -16,6 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Accelerating Apache Iceberg Parquet Scans using Comet
diff --git a/_sources/user-guide/0.15/installation.md.txt b/_sources/user-guide/0.15/installation.md.txt
index 655ddf96c..3e49bc472 100644
--- a/_sources/user-guide/0.15/installation.md.txt
+++ b/_sources/user-guide/0.15/installation.md.txt
@@ -16,6 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Installing DataFusion Comet
diff --git a/_sources/user-guide/0.15/kubernetes.md.txt b/_sources/user-guide/0.15/kubernetes.md.txt
index 2fb037d63..864881dfe 100644
--- a/_sources/user-guide/0.15/kubernetes.md.txt
+++ b/_sources/user-guide/0.15/kubernetes.md.txt
@@ -16,6 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Comet Kubernetes Support
diff --git a/_sources/user-guide/0.15/metrics.md.txt b/_sources/user-guide/0.15/metrics.md.txt
index 509d0ae8c..7caa948e6 100644
--- a/_sources/user-guide/0.15/metrics.md.txt
+++ b/_sources/user-guide/0.15/metrics.md.txt
@@ -16,6 +16,9 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Comet Metrics
diff --git a/_sources/user-guide/0.15/operators.md.txt b/_sources/user-guide/0.15/operators.md.txt
index 77ba84e4f..5b41c8879 100644
--- a/_sources/user-guide/0.15/operators.md.txt
+++ b/_sources/user-guide/0.15/operators.md.txt
@@ -16,6 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Supported Spark Operators
diff --git a/_sources/user-guide/0.15/source.md.txt b/_sources/user-guide/0.15/source.md.txt
index ed03cad5a..dbcff41f5 100644
--- a/_sources/user-guide/0.15/source.md.txt
+++ b/_sources/user-guide/0.15/source.md.txt
@@ -16,6 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Building Comet From Source
diff --git a/_sources/user-guide/0.15/tuning.md.txt b/_sources/user-guide/0.15/tuning.md.txt
index ff9acee1f..db219ab56 100644
--- a/_sources/user-guide/0.15/tuning.md.txt
+++ b/_sources/user-guide/0.15/tuning.md.txt
@@ -16,6 +16,9 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
+```{warning}
+This is **out-of-date** documentation. The latest Comet release is version 0.16.0.
+```
# Comet Tuning Guide
diff --git a/_sources/user-guide/0.16/compatibility/expressions/aggregate.md.txt b/_sources/user-guide/0.16/compatibility/expressions/aggregate.md.txt
new file mode 100644
index 000000000..317decbd7
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/expressions/aggregate.md.txt
@@ -0,0 +1,53 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Aggregate Expressions
+
+<!--BEGIN:EXPR_COMPAT[aggregate]-->
+
+## Average
+
+The following incompatibilities cause `Average` to fall back to Spark by default. Set `spark.comet.expression.Average.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Falls back to Spark in ANSI mode. Supports all numeric inputs except decimal types.
+
+## CollectSet
+
+The following incompatibilities cause `CollectSet` to fall back to Spark by default. Set `spark.comet.expression.CollectSet.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Comet deduplicates NaN values (treats `NaN == NaN`) while Spark treats each NaN as a distinct value. When `spark.comet.exec.strictFloatingPoint=true`, `collect_set` on floating-point types falls back to Spark unless `spark.comet.expression.CollectSet.allowIncompatible=true` is set.
+
+## First
+
+The following differences from Spark are always present and do not require any additional configuration:
+
+- This function is not deterministic. Results may not match Spark.
+
+## Last
+
+The following differences from Spark are always present and do not require any additional configuration:
+
+- This function is not deterministic. Results may not match Spark.
+
+## Sum
+
+The following incompatibilities cause `Sum` to fall back to Spark by default. Set `spark.comet.expression.Sum.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Falls back to Spark in ANSI mode.
+<!--END:EXPR_COMPAT-->
diff --git a/_sources/user-guide/0.16/compatibility/expressions/array.md.txt b/_sources/user-guide/0.16/compatibility/expressions/array.md.txt
new file mode 100644
index 000000000..d5f3eff81
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/expressions/array.md.txt
@@ -0,0 +1,79 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Array Expressions
+
+<!--BEGIN:EXPR_COMPAT[array]-->
+
+## ArrayExcept
+
+The following incompatibilities cause `ArrayExcept` to fall back to Spark by default. Set `spark.comet.expression.ArrayExcept.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Null handling and ordering may differ from Spark
+
+## ArrayFilter
+
+The following cases are not supported by Comet:
+
+- Only supports `array_filter` when the function is `IsNotNull` (used by `array_compact`)
+
+## ArrayIntersect
+
+The following incompatibilities cause `ArrayIntersect` to fall back to Spark by default. Set `spark.comet.expression.ArrayIntersect.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Result array element order may differ from Spark when the right array is longer than the left (DataFusion probes the longer side).
+
+The following cases are not supported by Comet:
+
+- array_intersect on collated strings is not supported.
+
+## ArrayJoin
+
+The following incompatibilities cause `ArrayJoin` to fall back to Spark by default. Set `spark.comet.expression.ArrayJoin.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Null handling may differ from Spark
+
+## ArraysZip
+
+The following cases are not supported by Comet:
+
+- Not all input data types are supported; falls back to Spark for unsupported types
+
+## ElementAt
+
+The following cases are not supported by Comet:
+
+- Input must be an array. `Map` inputs are not supported.
+
+## Size
+
+The following cases are not supported by Comet:
+
+- Only supports `ArrayType` input; `MapType` input is not supported
+
+## SortArray
+
+The following incompatibilities cause `SortArray` to fall back to Spark by default. Set `spark.comet.expression.SortArray.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- When `spark.comet.exec.strictFloatingPoint=true`, sorting on floating-point types is not 100% compatible with Spark
+
+The following cases are not supported by Comet:
+
+- Nested arrays with `Struct` or `Null` child values are not supported natively and will fall back to Spark.
+<!--END:EXPR_COMPAT-->
diff --git a/_sources/user-guide/0.16/compatibility/expressions/cast.md.txt b/_sources/user-guide/0.16/compatibility/expressions/cast.md.txt
new file mode 100644
index 000000000..f6d19dc3d
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/expressions/cast.md.txt
@@ -0,0 +1,242 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Cast
+
+Cast operations in Comet fall into three levels of support:
+
+- **C (Compatible)**: The results match Apache Spark
+- **I (Incompatible)**: The results may match Apache Spark for some inputs, but there are known issues where some inputs
+ will result in incorrect results or exceptions. The query stage will fall back to Spark by default. Setting
+ `spark.comet.expression.Cast.allowIncompatible=true` will allow all incompatible casts to run natively in Comet, but this is not
+ recommended for production use.
+- **U (Unsupported)**: Comet does not provide a native version of this cast expression and the query stage will fall back to
+ Spark.
+- **N/A**: Spark does not support this cast.
+
+## ANSI Mode Fallback
+
+Cast will fall back to Spark in some cases when ANSI mode is enabled. This can be enabled by setting `spark.comet.expression.Cast.allowIncompatible=true`. See the [Comet Supported Expressions Guide](../../expressions.md) for more information on this configuration setting.
+
+There is an [epic](https://github.com/apache/datafusion-comet/issues/313) where we are tracking the work to fully implement ANSI support.
+
+## String to Decimal
+
+Comet's native `CAST(string AS DECIMAL)` implementation matches Apache Spark's behavior,
+including:
+
+- Leading and trailing ASCII whitespace is trimmed before parsing.
+- Null bytes (`\u0000`) at the start or end of a string are trimmed, matching Spark's
+ `UTF8String` behavior. Null bytes embedded in the middle of a string produce `NULL`.
+- Fullwidth Unicode digits (U+FF10–U+FF19, e.g. `123.45`) are treated as their ASCII
+ equivalents, so `CAST('123.45' AS DECIMAL(10,2))` returns `123.45`.
+- Scientific notation (e.g. `1.23E+5`) is supported.
+- Special values (`inf`, `infinity`, `nan`) produce `NULL`.
+
+## String to Date
+
+Comet's native `CAST(string AS DATE)` implementation matches Apache Spark's behavior for years
+between 262143 BC and 262142 AD. This range limitation comes from the underlying chrono library's
+`NaiveDate` type. Spark itself supports a wider range. All three eval modes (Legacy, ANSI, Try)
+are supported.
+
+Supported input formats match Spark exactly:
+
+- `yyyy`, `yyyy-[m]m`, `yyyy-[m]m-[d]d`
+- Optional `T` suffix with arbitrary trailing text (e.g. `2020-01-01T12:34:56`)
+- Leading/trailing whitespace and control characters are trimmed
+- Optional sign prefix (`-` for negative years)
+- Leading zeros (e.g. `0002020-01-01` is year 2020)
+
+## Date to Timestamp
+
+Comet's native `CAST(date AS TIMESTAMP)` is compatible with Spark. The cast interprets each
+date as midnight in the session timezone and converts to a UTC epoch value. DST transitions
+are handled correctly, including spring-forward gaps (where midnight may not exist) and
+fall-back ambiguity (where Comet picks the earlier/DST occurrence, matching Spark's
+`LocalDate.atStartOfDay(zoneId)` behavior).
+
+## Date to TimestampNTZ
+
+Comet's native `CAST(date AS TIMESTAMP_NTZ)` is compatible with Spark. The cast is
+timezone-independent: each date is converted to midnight as pure arithmetic
+(`days * 86,400,000,000` microseconds) with no session timezone offset applied. The result
+is the same regardless of the session timezone setting.
+
+## Date to Numeric Types
+
+In Legacy mode, `CAST(date AS INT)`, `CAST(date AS LONG)`, and casts to all other numeric
+types (Boolean, Byte, Short, Float, Double, Decimal) always return `NULL`. Comet handles
+this by short-circuiting to a null literal during query planning, so no native execution
+is needed. In ANSI and Try modes, Spark rejects these casts at analysis time (before
+execution reaches Comet).
+
+## String to Timestamp
+
+Comet's native `CAST(string AS TIMESTAMP)` implementation supports all timestamp formats accepted
+by Apache Spark, including ISO 8601 date-time strings, date-only strings, time-only strings
+(`HH:MM:SS`), embedded timezone offsets (e.g. `+07:30`, `GMT-01:00`, `UTC`), named timezone
+suffixes (e.g. `Europe/Moscow`), and the full Spark timestamp year range
+(-290308 to 294247).
+
+## String to TimestampNTZ
+
+Comet's native `CAST(string AS TIMESTAMP_NTZ)` implementation matches Apache Spark's behavior.
+Unlike `CAST(string AS TIMESTAMP)`, this cast is timezone-independent: any timezone offset in
+the input string (e.g. `+08:00`, `Z`, `UTC`) is silently discarded, and the local date-time
+components are preserved as-is. Time-only strings (e.g. `T12:34:56`, `12:34`) produce `NULL`.
+The result is always a wall-clock timestamp with no timezone conversion or DST adjustment.
+
+## TimestampNTZ Casts
+
+Comet supports the following `TIMESTAMP_NTZ` casts natively:
+
+| Cast | Compatible | Notes |
+| ---------------------------------- | ---------- | ----------------------------------------------------------------- |
+| `CAST(timestamp_ntz AS STRING)` | Yes | Formats local time as-is, timezone-independent |
+| `CAST(timestamp_ntz AS DATE)` | Yes | Extracts the date component, timezone-independent |
+| `CAST(timestamp_ntz AS TIMESTAMP)` | Yes | Interprets NTZ as local time in session TZ, converts to UTC epoch |
+| `CAST(date AS TIMESTAMP_NTZ)` | Yes | Pure arithmetic, timezone-independent |
+| `CAST(timestamp AS TIMESTAMP_NTZ)` | Yes | Shifts UTC epoch to local time in session TZ |
+| `CAST(string AS TIMESTAMP_NTZ)` | Yes | See [String to TimestampNTZ](#string-to-timestampntz) above |
+
+The NTZ-to-Timestamp and Timestamp-to-NTZ casts are session-timezone-dependent (the session
+timezone determines the UTC offset). All other NTZ casts are timezone-independent and produce
+the same result regardless of the session timezone.
+
+## Date to String
+
+Comet's native `CAST(date AS STRING)` is compatible with Spark. Years below 1000 are
+zero-padded to four digits (e.g. year 999 renders as `0999-01-01`). Years above 9999 are
+rendered without truncation. The cast is timezone-independent.
+
+## String to TimestampNTZ
+
+Comet's native `CAST(string AS TIMESTAMP_NTZ)` implementation matches Apache Spark's behavior.
+Unlike `CAST(string AS TIMESTAMP)`, this cast is timezone-independent: any timezone offset in
+the input string (e.g. `+08:00`, `Z`, `UTC`) is silently discarded, and the local date-time
+components are preserved as-is. Time-only strings (e.g. `T12:34:56`, `12:34`) produce `NULL`.
+The result is always a wall-clock timestamp with no timezone conversion or DST adjustment.
+
+## Decimal with Negative Scale to String
+
+Casting a `DecimalType` with a negative scale to `StringType` is marked as incompatible when
+`spark.sql.legacy.allowNegativeScaleOfDecimal` is `false` (the default). When that config is
+disabled, Spark cannot create negative-scale decimals, so Comet falls back to avoid running
+native execution on unexpected inputs.
+
+When `spark.sql.legacy.allowNegativeScaleOfDecimal=true`, the cast is compatible. Comet matches
+Spark's behavior of using Java `BigDecimal.toString()` semantics, which produces scientific
+notation (e.g. a value of 12300 stored as `Decimal(7,-2)` with unscaled value 123 is rendered
+as `"1.23E+4"`).
+
+## Legacy Mode
+
+<!--BEGIN:CAST_LEGACY_TABLE-->
+<!-- prettier-ignore-start -->
+
+| | binary | boolean | byte | date | decimal | double | float | integer | long | short | string | timestamp | timestamp_ntz |
+| ------------- | ------ | ------- | ---- | ---- | ------- | ------ | ----- | ------- | ---- | ----- | ------ | --------- | ------------- |
+| binary | - | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | C | N/A | N/A |
+| boolean | N/A | - | C | N/A | C | C | C | C | C | C | C | C | N/A |
+| byte | C | C | - | N/A | C | C | C | C | C | C | C | C | N/A |
+| date | N/A | C | C | - | C | C | C | C | C | C | C | C | C |
+| decimal | N/A | C | C | N/A | - | C | C | C | C | C | C | C | N/A |
+| double | N/A | C | C | N/A | I | - | C | C | C | C | C | C | N/A |
+| float | N/A | C | C | N/A | I | C | - | C | C | C | C | C | N/A |
+| integer | C | C | C | N/A | C | C | C | - | C | C | C | C | N/A |
+| long | C | C | C | N/A | C | C | C | C | - | C | C | C | N/A |
+| short | C | C | C | N/A | C | C | C | C | C | - | C | C | N/A |
+| string | C | C | C | C | C | C | C | C | C | C | - | C | C |
+| timestamp | N/A | U | U | C | U | U | U | U | C | U | C | - | C |
+| timestamp_ntz | N/A | N/A | N/A | C | N/A | N/A | N/A | N/A | N/A | N/A | C | C | - |
+
+**Notes:**
+
+- **double -> decimal**: There can be rounding differences
+- **double -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
+- **float -> decimal**: There can be rounding differences
+- **float -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
+- **string -> date**: Only supports years between 262143 BC and 262142 AD
+ <!-- prettier-ignore-end -->
+ <!--END:CAST_LEGACY_TABLE-->
+
+## Try Mode
+
+<!--BEGIN:CAST_TRY_TABLE-->
+<!-- prettier-ignore-start -->
+
+| | binary | boolean | byte | date | decimal | double | float | integer | long | short | string | timestamp | timestamp_ntz |
+| ------------- | ------ | ------- | ---- | ---- | ------- | ------ | ----- | ------- | ---- | ----- | ------ | --------- | ------------- |
+| binary | - | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | C | N/A | N/A |
+| boolean | N/A | - | C | N/A | C | C | C | C | C | C | C | U | N/A |
+| byte | U | C | - | N/A | C | C | C | C | C | C | C | C | N/A |
+| date | N/A | U | U | - | U | U | U | U | U | U | C | C | C |
+| decimal | N/A | C | C | N/A | - | C | C | C | C | C | C | C | N/A |
+| double | N/A | C | C | N/A | I | - | C | C | C | C | C | C | N/A |
+| float | N/A | C | C | N/A | I | C | - | C | C | C | C | C | N/A |
+| integer | U | C | C | N/A | C | C | C | - | C | C | C | C | N/A |
+| long | U | C | C | N/A | C | C | C | C | - | C | C | C | N/A |
+| short | U | C | C | N/A | C | C | C | C | C | - | C | C | N/A |
+| string | C | C | C | C | C | C | C | C | C | C | - | C | C |
+| timestamp | N/A | U | U | C | U | U | U | U | C | U | C | - | C |
+| timestamp_ntz | N/A | N/A | N/A | C | N/A | N/A | N/A | N/A | N/A | N/A | C | C | - |
+
+**Notes:**
+
+- **double -> decimal**: There can be rounding differences
+- **double -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
+- **float -> decimal**: There can be rounding differences
+- **float -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
+- **string -> date**: Only supports years between 262143 BC and 262142 AD
+ <!-- prettier-ignore-end -->
+ <!--END:CAST_TRY_TABLE-->
+
+## ANSI Mode
+
+<!--BEGIN:CAST_ANSI_TABLE-->
+<!-- prettier-ignore-start -->
+
+| | binary | boolean | byte | date | decimal | double | float | integer | long | short | string | timestamp | timestamp_ntz |
+| ------------- | ------ | ------- | ---- | ---- | ------- | ------ | ----- | ------- | ---- | ----- | ------ | --------- | ------------- |
+| binary | - | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | C | N/A | N/A |
+| boolean | N/A | - | C | N/A | C | C | C | C | C | C | C | U | N/A |
+| byte | U | C | - | N/A | C | C | C | C | C | C | C | C | N/A |
+| date | N/A | U | U | - | U | U | U | U | U | U | C | C | C |
+| decimal | N/A | C | C | N/A | - | C | C | C | C | C | C | C | N/A |
+| double | N/A | C | C | N/A | I | - | C | C | C | C | C | C | N/A |
+| float | N/A | C | C | N/A | I | C | - | C | C | C | C | C | N/A |
+| integer | U | C | C | N/A | C | C | C | - | C | C | C | C | N/A |
+| long | U | C | C | N/A | C | C | C | C | - | C | C | C | N/A |
+| short | U | C | C | N/A | C | C | C | C | C | - | C | C | N/A |
+| string | C | C | C | C | C | C | C | C | C | C | - | C | C |
+| timestamp | N/A | U | U | C | U | U | U | U | C | U | C | - | C |
+| timestamp_ntz | N/A | N/A | N/A | C | N/A | N/A | N/A | N/A | N/A | N/A | C | C | - |
+
+**Notes:**
+
+- **double -> decimal**: There can be rounding differences
+- **double -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
+- **float -> decimal**: There can be rounding differences
+- **float -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
+- **string -> date**: Only supports years between 262143 BC and 262142 AD
+ <!-- prettier-ignore-end -->
+ <!--END:CAST_ANSI_TABLE-->
+
+See the [tracking issue](https://github.com/apache/datafusion-comet/issues/286) for more details.
diff --git a/_sources/user-guide/0.16/compatibility/expressions/datetime.md.txt b/_sources/user-guide/0.16/compatibility/expressions/datetime.md.txt
new file mode 100644
index 000000000..612bc6ead
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/expressions/datetime.md.txt
@@ -0,0 +1,125 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Date/Time Expressions
+
+- **Hour, Minute, Second**: Incorrectly apply timezone conversion to TimestampNTZ inputs. TimestampNTZ stores local
+ time without timezone, so no conversion should be applied. These expressions work correctly with Timestamp inputs.
+ [#3180](https://github.com/apache/datafusion-comet/issues/3180)
+- **TruncTimestamp (date_trunc)**: Produces incorrect results when used with non-UTC timezones. Compatible when
+ timezone is UTC. TimestampNTZ inputs are handled correctly (timezone-independent truncation).
+ [#2649](https://github.com/apache/datafusion-comet/issues/2649)
+
+## Date and Time Functions
+
+Comet's native implementation of date and time functions may produce different results than Spark for dates
+far in the future (approximately beyond year 2100). This is because Comet uses the chrono-tz library for
+timezone calculations, which has limited support for Daylight Saving Time (DST) rules beyond the IANA
+time zone database's explicit transitions.
+
+For dates within a reasonable range (approximately 1970-2100), Comet's date and time functions are compatible
+with Spark. For dates beyond this range, functions that involve timezone-aware calculations (such as
+`date_trunc` with timezone-aware timestamps) may produce results with incorrect DST offsets.
+
+If you need to process dates far in the future with accurate timezone handling, consider:
+
+- Using timezone-naive types (`timestamp_ntz`) when timezone conversion is not required
+- Falling back to Spark for these specific operations
+
+<!--BEGIN:EXPR_COMPAT[datetime]-->
+
+## DateFormatClass
+
+The following incompatibilities cause `DateFormatClass` to fall back to Spark by default. Set `spark.comet.expression.DateFormatClass.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Non-UTC timezones may produce different results than Spark
+
+The following cases are not supported by Comet:
+
+- Only the following formats are supported:
+ - `EEE`
+ - `EEEE`
+ - `HH`
+ - `HH:mm`
+ - `HH:mm:ss`
+ - `MM`
+ - `MMM`
+ - `MMMM`
+ - `dd`
+ - `h:mm a`
+ - `hh:mm a`
+ - `hh:mm:ss a`
+ - `mm`
+ - `ss`
+ - `yy`
+ - `yyyy`
+ - `yyyy-MM-dd`
+ - `yyyy-MM-dd HH:mm:ss`
+ - `yyyy-MM-dd'T'HH:mm:ss`
+ - `yyyy/MM/dd`
+ - `yyyy/MM/dd HH:mm:ss`
+ - `yyyyMM`
+ - `yyyyMMdd`
+
+## FromUnixTime
+
+The following incompatibilities cause `FromUnixTime` to fall back to Spark by default. Set `spark.comet.expression.FromUnixTime.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Only supports the default datetime format pattern `yyyy-MM-dd HH:mm:ss`. DataFusion's valid timestamp range differs from Spark (https://github.com/apache/datafusion/issues/16594)
+
+## Hour
+
+The following incompatibilities cause `Hour` to fall back to Spark by default. Set `spark.comet.expression.Hour.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Incorrectly applies timezone conversion to TimestampNTZ inputs (https://github.com/apache/datafusion-comet/issues/3180)
+
+## Minute
+
+The following incompatibilities cause `Minute` to fall back to Spark by default. Set `spark.comet.expression.Minute.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Incorrectly applies timezone conversion to TimestampNTZ inputs (https://github.com/apache/datafusion-comet/issues/3180)
+
+## Second
+
+The following incompatibilities cause `Second` to fall back to Spark by default. Set `spark.comet.expression.Second.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Incorrectly applies timezone conversion to TimestampNTZ inputs (https://github.com/apache/datafusion-comet/issues/3180)
+
+## TruncDate
+
+The following incompatibilities cause `TruncDate` to fall back to Spark by default. Set `spark.comet.expression.TruncDate.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Non-literal format strings will throw an exception instead of returning NULL
+
+The following cases are not supported by Comet:
+
+- Only the following formats are supported: year, yyyy, yy, quarter, mon, month, mm, week
+
+## TruncTimestamp
+
+The following incompatibilities cause `TruncTimestamp` to fall back to Spark by default. Set `spark.comet.expression.TruncTimestamp.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Produces incorrect results when used with non-UTC timezones. Compatible when timezone is UTC. (https://github.com/apache/datafusion-comet/issues/2649)
+
+## UnixTimestamp
+
+The following cases are not supported by Comet:
+
+- Only `TimestampType` and `DateType` inputs are supported. `TimestampNTZType` is not supported because Comet incorrectly applies timezone conversion to TimestampNTZ values.
+<!--END:EXPR_COMPAT-->
diff --git a/_sources/user-guide/index.md.txt b/_sources/user-guide/0.16/compatibility/expressions/index.md.txt
similarity index 54%
copy from _sources/user-guide/index.md.txt
copy to _sources/user-guide/0.16/compatibility/expressions/index.md.txt
index 65736c1d4..ba0c4b5d5 100644
--- a/_sources/user-guide/index.md.txt
+++ b/_sources/user-guide/0.16/compatibility/expressions/index.md.txt
@@ -1,4 +1,4 @@
-<!--
+<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -7,7 +7,7 @@ to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
@@ -17,23 +17,25 @@ specific language governing permissions and limitations
under the License.
-->
-# Comet User Guide
+# Expression Compatibility
-The Comet user guide covers installation, configuration, supported data sources, supported operators
-and expressions, and tuning advice for running Apache Spark with Comet acceleration.
+Expressions that are not 100% Spark-compatible will fall back to Spark by default and can be enabled by setting
+`spark.comet.expression.EXPRNAME.allowIncompatible=true`, where `EXPRNAME` is the Spark expression class name. See
+the [Comet Supported Expressions Guide](../../expressions.md) for more information on this configuration setting.
-User guides are published for each release. The development snapshot tracks the upcoming release and
-may include features and fixes that are not yet generally available. Select a version from the
-navigation menu to view its guide.
+Compatibility notes are grouped by expression category:
```{toctree}
-:maxdepth: 2
-:caption: User Guides
+:maxdepth: 1
:hidden:
-0.16.0-SNAPSHOT <latest/index>
-0.15.x <0.15/index>
-0.14.x <0.14/index>
-0.13.x <0.13/index>
-0.12.x <0.12/index>
+aggregate
+array
+datetime
+map
+math
+misc
+string
+struct
+cast
```
diff --git a/_sources/user-guide/0.16/compatibility/expressions/map.md.txt b/_sources/user-guide/0.16/compatibility/expressions/map.md.txt
new file mode 100644
index 000000000..ec9fd6391
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/expressions/map.md.txt
@@ -0,0 +1,41 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Map Expressions
+
+## MapSort (Spark 4.0+)
+
+Spark 4.0 inserts `MapSort` to normalize map values when they appear in shuffle hash partitioning
+keys, in `try_element_at`, and in other contexts where map ordering must be deterministic. Comet
+runs `MapSort` natively, so map shuffle and group-by-on-map stay on Comet under Spark 4.0.
+
+When `spark.comet.exec.strictFloatingPoint=true`, `MapSort` falls back to Spark for maps whose
+keys contain `Float` or `Double` (consistent with `SortOrder` and `SortArray`). Arrow's sort uses
+IEEE total ordering for floating-point, which differs from Spark's `Double.compare` semantics for
+`NaN` and `-0.0`.
+
+<!--BEGIN:EXPR_COMPAT[map]-->
+
+## MapFromEntries
+
+The following incompatibilities cause `MapFromEntries` to fall back to Spark by default. Set `spark.comet.expression.MapFromEntries.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Using BinaryType as Map keys is not allowed in map_from_entries
+- Using BinaryType as Map values is not allowed in map_from_entries
+<!--END:EXPR_COMPAT-->
diff --git a/_sources/user-guide/index.md.txt b/_sources/user-guide/0.16/compatibility/expressions/math.md.txt
similarity index 52%
copy from _sources/user-guide/index.md.txt
copy to _sources/user-guide/0.16/compatibility/expressions/math.md.txt
index 65736c1d4..f347f697e 100644
--- a/_sources/user-guide/index.md.txt
+++ b/_sources/user-guide/0.16/compatibility/expressions/math.md.txt
@@ -1,4 +1,4 @@
-<!--
+<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -7,7 +7,7 @@ to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
@@ -17,23 +17,13 @@ specific language governing permissions and limitations
under the License.
-->
-# Comet User Guide
+# Math Expressions
-The Comet user guide covers installation, configuration, supported data sources, supported operators
-and expressions, and tuning advice for running Apache Spark with Comet acceleration.
+<!--BEGIN:EXPR_COMPAT[math]-->
-User guides are published for each release. The development snapshot tracks the upcoming release and
-may include features and fixes that are not yet generally available. Select a version from the
-navigation menu to view its guide.
+## Abs
-```{toctree}
-:maxdepth: 2
-:caption: User Guides
-:hidden:
+The following cases are not supported by Comet:
-0.16.0-SNAPSHOT <latest/index>
-0.15.x <0.15/index>
-0.14.x <0.14/index>
-0.13.x <0.13/index>
-0.12.x <0.12/index>
-```
+- Only integral, floating-point, and decimal types are supported
+<!--END:EXPR_COMPAT-->
diff --git a/_sources/user-guide/0.16/compatibility/expressions/misc.md.txt b/_sources/user-guide/0.16/compatibility/expressions/misc.md.txt
new file mode 100644
index 000000000..185282dcd
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/expressions/misc.md.txt
@@ -0,0 +1,53 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Miscellaneous Expressions
+
+<!--BEGIN:EXPR_COMPAT[misc]-->
+
+## CheckOverflow
+
+The following cases are not supported by Comet:
+
+- Only `DecimalType` is supported
+
+## KnownFloatingPointNormalized
+
+The following cases are not supported by Comet:
+
+- Only supports `NormalizeNaNAndZero` child expressions
+
+## Literal
+
+The following cases are not supported by Comet:
+
+- Not all data types are supported for literal values
+
+## MakeDecimal
+
+The following cases are not supported by Comet:
+
+- Only `LongType` input is supported
+
+## SortOrder
+
+The following incompatibilities cause `SortOrder` to fall back to Spark by default. Set `spark.comet.expression.SortOrder.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- When `spark.comet.exec.strictFloatingPoint=true`, sorting on floating-point types is not 100% compatible with Spark
+<!--END:EXPR_COMPAT-->
diff --git a/_sources/user-guide/0.16/compatibility/expressions/string.md.txt b/_sources/user-guide/0.16/compatibility/expressions/string.md.txt
new file mode 100644
index 000000000..dc7466776
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/expressions/string.md.txt
@@ -0,0 +1,118 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# String Expressions
+
+<!--BEGIN:EXPR_COMPAT[string]-->
+
+## Concat
+
+The following incompatibilities cause `Concat` to fall back to Spark by default. Set `spark.comet.expression.Concat.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- CONCAT supports only string input parameters
+
+## GetJsonObject
+
+The following incompatibilities cause `GetJsonObject` to fall back to Spark by default. Set `spark.comet.expression.GetJsonObject.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Spark allows single-quoted JSON and unescaped control characters which Comet does not support
+
+## InitCap
+
+The following incompatibilities cause `InitCap` to fall back to Spark by default. Set `spark.comet.expression.InitCap.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Treats hyphen as a word separator (e.g. `robert rose-smith` produces `Robert Rose-Smith` instead of Spark's `Robert Rose-smith`) (https://github.com/apache/datafusion-comet/issues/1052)
+
+## Left
+
+The following cases are not supported by Comet:
+
+- Only supports `BinaryType` and `StringType` input
+- The length argument must be a literal value
+
+## Length
+
+The following cases are not supported by Comet:
+
+- `BinaryType` input is not supported
+
+## Lower
+
+The following incompatibilities cause `Lower` to fall back to Spark by default. Set `spark.comet.expression.Lower.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Results can vary depending on locale and character set. Requires `spark.comet.caseConversion.enabled=true` to enable.
+
+## RLike
+
+The following incompatibilities cause `RLike` to fall back to Spark by default. Set `spark.comet.expression.RLike.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Uses Rust regexp engine, which has different behavior to Java regexp engine
+
+## RegExpReplace
+
+The following incompatibilities cause `RegExpReplace` to fall back to Spark by default. Set `spark.comet.expression.RegExpReplace.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Regexp pattern may not be compatible with Spark
+
+The following cases are not supported by Comet:
+
+- Only supports `regexp_replace` with an offset of 1 (no offset)
+
+## Reverse
+
+The following incompatibilities cause `Reverse` to fall back to Spark by default. Set `spark.comet.expression.Reverse.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- reverse on array containing binary is not supported
+
+## Right
+
+The following cases are not supported by Comet:
+
+- Only supports `StringType` input
+
+## StringLPad
+
+The following cases are not supported by Comet:
+
+- Scalar values are not supported for the `str` argument. Only scalar values are supported for the `pad` argument.
+
+## StringRPad
+
+The following cases are not supported by Comet:
+
+- Scalar values are not supported for the `str` argument. Only scalar values are supported for the `pad` argument.
+
+## StringRepeat
+
+The following differences from Spark are always present and do not require any additional configuration:
+
+- A negative argument for the number of times to repeat throws an exception instead of returning an empty string as Spark does
+
+## StringSplit
+
+The following incompatibilities cause `StringSplit` to fall back to Spark by default. Set `spark.comet.expression.StringSplit.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Regex engine differences between Java and Rust
+
+## Upper
+
+The following incompatibilities cause `Upper` to fall back to Spark by default. Set `spark.comet.expression.Upper.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Results can vary depending on locale and character set. Requires `spark.comet.caseConversion.enabled=true` to enable.
+<!--END:EXPR_COMPAT-->
diff --git a/_sources/user-guide/0.16/compatibility/expressions/struct.md.txt b/_sources/user-guide/0.16/compatibility/expressions/struct.md.txt
new file mode 100644
index 000000000..3c082ef8b
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/expressions/struct.md.txt
@@ -0,0 +1,49 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Struct Expressions
+
+<!--BEGIN:EXPR_COMPAT[struct]-->
+
+## JsonToStructs
+
+The following incompatibilities cause `JsonToStructs` to fall back to Spark by default. Set `spark.comet.expression.JsonToStructs.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Partially implemented and not comprehensively tested
+
+The following cases are not supported by Comet:
+
+- Requires an explicit schema
+
+## StructsToCsv
+
+The following incompatibilities cause `StructsToCsv` to fall back to Spark by default. Set `spark.comet.expression.StructsToCsv.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Date, Timestamp, TimestampNTZ, and Binary data types may produce different results (https://github.com/apache/datafusion-comet/issues/3232)
+
+The following cases are not supported by Comet:
+
+- Complex types (arrays, maps, structs) in the schema are not supported
+
+## StructsToJson
+
+The following incompatibilities cause `StructsToJson` to fall back to Spark by default. Set `spark.comet.expression.StructsToJson.allowIncompatible=true` to enable Comet acceleration despite these differences.
+
+- Does not support `+Infinity` and `-Infinity` for numeric types (float, double). (https://github.com/apache/datafusion-comet/issues/3016)
+<!--END:EXPR_COMPAT-->
diff --git a/_sources/user-guide/0.16/compatibility/floating-point.md.txt b/_sources/user-guide/0.16/compatibility/floating-point.md.txt
new file mode 100644
index 000000000..ffab05506
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/floating-point.md.txt
@@ -0,0 +1,29 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Floating-point Number Comparison
+
+Spark normalizes NaN and zero for floating point numbers for several cases. See `NormalizeFloatingNumbers` optimization rule in Spark.
+However, one exception is comparison. Spark does not normalize NaN and zero when comparing values
+because they are handled well in Spark (e.g., `SQLOrderingUtil.compareFloats`). But the comparison
+functions of arrow-rs used by DataFusion do not normalize NaN and zero (e.g., [arrow::compute::kernels::cmp::eq](https://docs.rs/arrow/latest/arrow/compute/kernels/cmp/fn.eq.html#)).
+So Comet adds additional normalization expression of NaN and zero for comparisons, and may still have differences
+to Spark in some cases, especially when the data contains both positive and negative zero. This is likely an edge
+case that is not of concern for many users. If it is a concern, setting `spark.comet.exec.strictFloatingPoint=true`
+will make relevant operations fall back to Spark.
diff --git a/_sources/user-guide/0.16/compatibility/index.md.txt b/_sources/user-guide/0.16/compatibility/index.md.txt
new file mode 100644
index 000000000..1ba9d9e18
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/index.md.txt
@@ -0,0 +1,43 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Compatibility Guide
+
+Comet aims to provide consistent results with the version of Apache Spark that is being used.
+
+This guide documents areas where Comet's behavior is known to differ from Spark. Topics are grouped by subsystem:
+
+- **Parquet**: limitations when reading Parquet files (both scan implementations, shared and per-implementation).
+- **Floating-point comparison**: NaN and signed-zero handling in comparisons.
+- **Regular expressions**: differences between the Rust regexp crate and Java's regex engine.
+- **Operators**: operator-level compatibility notes, including window functions and round-robin partitioning.
+- **Expressions**: per-expression compatibility notes, including cast.
+- **Spark versions**: version-specific known issues and limitations.
+
+```{toctree}
+:maxdepth: 1
+:hidden:
+
+scans
+floating-point
+regex
+operators
+expressions/index
+spark-versions
+```
diff --git a/_sources/user-guide/0.16/compatibility/operators.md.txt b/_sources/user-guide/0.16/compatibility/operators.md.txt
new file mode 100644
index 000000000..a25d1d1db
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/operators.md.txt
@@ -0,0 +1,51 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Operator Compatibility
+
+## Window Functions
+
+Comet's support for window functions is incomplete and known to be incorrect. It is disabled by default and
+should not be used in production. The feature will be enabled in a future release. Tracking issue: [#2721](https://github.com/apache/datafusion-comet/issues/2721).
+
+## Round-Robin Partitioning
+
+Comet's native shuffle implementation of round-robin partitioning (`df.repartition(n)`) is not compatible with
+Spark's implementation and is disabled by default. It can be enabled by setting
+`spark.comet.native.shuffle.partitioning.roundrobin.enabled=true`.
+
+**Why the incompatibility exists:**
+
+Spark's round-robin partitioning sorts rows by their binary `UnsafeRow` representation before assigning them to
+partitions. This ensures deterministic output for fault tolerance (task retries produce identical results).
+Comet uses Arrow format internally, which has a completely different binary layout than `UnsafeRow`, making it
+impossible to match Spark's exact partition assignments.
+
+**Comet's approach:**
+
+Instead of true round-robin assignment, Comet implements round-robin as hash partitioning on ALL columns. This
+achieves the same semantic goals:
+
+- **Even distribution**: Rows are distributed evenly across partitions (as long as the hash varies sufficiently -
+ in some cases there could be skew)
+- **Deterministic**: Same input always produces the same partition assignments (important for fault tolerance)
+- **No semantic grouping**: Unlike hash partitioning on specific columns, this doesn't group related rows together
+
+The only difference is that Comet's partition assignments will differ from Spark's. When results are sorted,
+they will be identical to Spark. Unsorted results may have different row ordering.
diff --git a/_sources/user-guide/index.md.txt b/_sources/user-guide/0.16/compatibility/regex.md.txt
similarity index 51%
copy from _sources/user-guide/index.md.txt
copy to _sources/user-guide/0.16/compatibility/regex.md.txt
index 65736c1d4..4d9d5b650 100644
--- a/_sources/user-guide/index.md.txt
+++ b/_sources/user-guide/0.16/compatibility/regex.md.txt
@@ -1,4 +1,4 @@
-<!--
+<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -7,7 +7,7 @@ to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
@@ -17,23 +17,8 @@ specific language governing permissions and limitations
under the License.
-->
-# Comet User Guide
+# Regular Expressions
-The Comet user guide covers installation, configuration, supported data sources, supported operators
-and expressions, and tuning advice for running Apache Spark with Comet acceleration.
-
-User guides are published for each release. The development snapshot tracks the upcoming release and
-may include features and fixes that are not yet generally available. Select a version from the
-navigation menu to view its guide.
-
-```{toctree}
-:maxdepth: 2
-:caption: User Guides
-:hidden:
-
-0.16.0-SNAPSHOT <latest/index>
-0.15.x <0.15/index>
-0.14.x <0.14/index>
-0.13.x <0.13/index>
-0.12.x <0.12/index>
-```
+Comet uses the Rust regexp crate for evaluating regular expressions, and this has different behavior from Java's
+regular expression engine. Comet will fall back to Spark for patterns that are known to produce different results, but
+this can be overridden by setting `spark.comet.expression.regexp.allowIncompatible=true`.
diff --git a/_sources/user-guide/0.16/compatibility/scans.md.txt b/_sources/user-guide/0.16/compatibility/scans.md.txt
new file mode 100644
index 000000000..d713dd4e8
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/scans.md.txt
@@ -0,0 +1,108 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Parquet Compatibility
+
+Comet currently has two distinct implementations of the Parquet scan operator.
+
+| Scan Implementation | Notes |
+| ----------------------- | ---------------------- |
+| `native_datafusion` | Fully native scan |
+| `native_iceberg_compat` | Hybrid JVM/native scan |
+
+The configuration property `spark.comet.scan.impl` is used to select an implementation. The default setting is
+`spark.comet.scan.impl=auto`, which attempts to use `native_datafusion` first, and falls back to Spark if the scan
+cannot be converted (e.g., due to unsupported features). Most users should not need to change this setting. However,
+it is possible to force Comet to use a particular implementation for all scan operations by setting this
+configuration property to one of the following implementations. For example: `--conf spark.comet.scan.impl=native_datafusion`.
+
+## Shared Limitations
+
+The following features are not supported by either scan implementation, and Comet will fall back to Spark in these scenarios:
+
+- Decimals encoded in binary format.
+- `ShortType` columns, by default. When reading Parquet files written by systems other than Spark that contain
+ columns with the logical type `UINT_8` (unsigned 8-bit integers), Comet may produce different results than Spark.
+ Spark maps `UINT_8` to `ShortType`, but Comet's Arrow-based readers respect the unsigned type and read the data as
+ unsigned rather than signed. Since Comet cannot distinguish `ShortType` columns that came from `UINT_8` versus
+ signed `INT16`, by default Comet falls back to Spark when scanning Parquet files containing `ShortType` columns.
+ This behavior can be disabled by setting `spark.comet.scan.unsignedSmallIntSafetyCheck=false`. Note that `ByteType`
+ columns are always safe because they can only come from signed `INT8`, where truncation preserves the signed value.
+- Default values that are nested types (e.g., maps, arrays, structs). Literal default values are supported.
+- Spark's Datasource V2 API. When `spark.sql.sources.useV1SourceList` does not include `parquet`, Spark uses the
+ V2 API for Parquet scans. The DataFusion-based implementations only support the V1 API.
+- Spark metadata columns (e.g., `_metadata.file_path`)
+
+The following shared limitation may produce incorrect results without falling back to Spark:
+
+- No support for datetime rebasing. When reading Parquet files containing dates or timestamps written before
+ Spark 3.0 (which used a hybrid Julian/Gregorian calendar), dates/timestamps will be read as if they were
+ written using the Proleptic Gregorian calendar. This may produce incorrect results for dates before
+ October 15, 1582.
+
+The following shared limitation raises an error at scan time rather than falling back to Spark:
+
+- Invalid UTF-8 bytes in `STRING` columns. Spark permits arbitrary byte sequences in a `STRING`
+ column (for example from `CAST(X'C1' AS STRING)`), but Comet's native execution path is built on
+ Arrow, whose string type is strictly UTF-8. Reading a Parquet file whose `STRING` column contains
+ non-UTF-8 bytes fails with `Parquet error: encountered non UTF-8 data`. Disable Comet for the
+ query, or cast the column to `BINARY` before persisting, if you need to preserve non-UTF-8 bytes.
+ See [#4121](https://github.com/apache/datafusion-comet/issues/4121).
+
+## `native_datafusion` Limitations
+
+The `native_datafusion` scan has some additional limitations, mostly related to Parquet metadata. All of these
+cause Comet to fall back to Spark (including when using `auto` mode). Note that the `native_datafusion` scan
+requires `spark.comet.exec.enabled=true` because the scan node must be wrapped by `CometExecRule`.
+
+- No support for row indexes
+- No support for reading Parquet field IDs
+- No support for `input_file_name()`, `input_file_block_start()`, or `input_file_block_length()` SQL functions.
+ The `native_datafusion` scan does not use Spark's `FileScanRDD`, so these functions cannot populate their values.
+- No support for `ignoreMissingFiles` or `ignoreCorruptFiles` being set to `true`
+- Duplicate field names in case-insensitive mode (e.g., a Parquet file with both `B` and `b` columns)
+ are detected at read time and raise a `SparkRuntimeException` with error class `_LEGACY_ERROR_TEMP_2093`,
+ matching Spark's behavior.
+
+The following `native_datafusion` limitations may produce incorrect results on Spark versions prior to 4.0
+without falling back to Spark:
+
+- Reading `TimestampLTZ` as `TimestampNTZ`. On Spark 3.x, Spark raises an error per
+ [SPARK-36182](https://issues.apache.org/jira/browse/SPARK-36182) because LTZ encodes UTC-adjusted instants
+ that cannot be safely reinterpreted as timezone-free values. Comet does not raise this error and instead
+ returns the raw UTC instant as a `TimestampNTZ` value. This applies to all LTZ physical encodings (INT96,
+ TIMESTAMP_MICROS, TIMESTAMP_MILLIS). On Spark 4.0+, this read is permitted
+ ([SPARK-47447](https://issues.apache.org/jira/browse/SPARK-47447)) and Comet matches Spark's behavior.
+ See [#4219](https://github.com/apache/datafusion-comet/issues/4219).
+
+- Unsupported Parquet type conversions. Spark raises schema incompatibility errors for certain conversions
+ (e.g., reading INT32 as BIGINT, reading BINARY as TIMESTAMP, unsupported decimal precision changes).
+ The `native_datafusion` scan may not detect these mismatches and could return unexpected values instead
+ of raising an error. See [#3720](https://github.com/apache/datafusion-comet/issues/3720).
+
+## `native_iceberg_compat` Limitations
+
+The `native_iceberg_compat` scan has the following additional limitation that may produce incorrect results
+without falling back to Spark:
+
+- Some Spark configuration values are hard-coded to their defaults rather than respecting user-specified values.
+ This may produce incorrect results when non-default values are set. The affected configurations are
+ `spark.sql.parquet.binaryAsString`, `spark.sql.parquet.int96AsTimestamp`, `spark.sql.caseSensitive`,
+ `spark.sql.parquet.inferTimestampNTZ.enabled`, and `spark.sql.legacy.parquet.nanosAsLong`. See
+ [issue #1816](https://github.com/apache/datafusion-comet/issues/1816) for more details.
diff --git a/_sources/user-guide/0.16/compatibility/spark-versions.md.txt b/_sources/user-guide/0.16/compatibility/spark-versions.md.txt
new file mode 100644
index 000000000..663be37a2
--- /dev/null
+++ b/_sources/user-guide/0.16/compatibility/spark-versions.md.txt
@@ -0,0 +1,79 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Spark Version Compatibility
+
+This page documents known issues and limitations specific to each supported Apache Spark version.
+
+For general compatibility information that applies across all Spark versions, see the other pages in this
+compatibility guide.
+
+## Spark 3.4
+
+Spark 3.4.3 is supported with Java 11/17 and Scala 2.12/2.13.
+
+### Known Limitations
+
+- **Reading `TimestampLTZ` as `TimestampNTZ`**: Spark 3.4 raises an error for this operation
+ (SPARK-36182), but Comet's `native_datafusion` scan silently returns the raw UTC value instead.
+ See [Parquet Compatibility](scans.md#native_datafusion-limitations) for details.
+
+## Spark 3.5
+
+Spark 3.5.8 is supported with Java 11/17 and Scala 2.12/2.13.
+
+### Known Limitations
+
+- **Reading `TimestampLTZ` as `TimestampNTZ`**: Spark 3.5 raises an error for this operation
+ (SPARK-36182), but Comet's `native_datafusion` scan silently returns the raw UTC value instead.
+ See [Parquet Compatibility](scans.md#native_datafusion-limitations) for details.
+
+## Spark 4.0
+
+Spark 4.0.2 is supported with Java 17 and Scala 2.13.
+
+### Known Limitations
+
+- **Collation support** ([#1947](https://github.com/apache/datafusion-comet/issues/1947),
+ [#4051](https://github.com/apache/datafusion-comet/issues/4051)): Spark 4.0 introduced collation
+ support. Non-default collated strings are not yet supported by Comet and will fall back to Spark.
+
+## Spark 4.1
+
+Spark 4.1.1 is supported with Java 17/21 and Scala 2.13.
+
+### Known Limitations
+
+- **`NullType` columns in Parquet files**
+ ([#4199](https://github.com/apache/datafusion-comet/issues/4199)): Spark encodes a `NullType`
+ column as a Parquet `BOOLEAN` physical type annotated with `LogicalType::Unknown`. The Rust
+ `parquet` crate that Comet depends on accepts `Unknown` only when paired with `INT32` and rejects
+ any other physical type with `Parquet error: Cannot annotate Unknown from BOOLEAN for field '<name>'`.
+ Any attempt to read a Parquet file that contains a `NullType` column fails at decode time before
+ Comet's scan runs. Workaround: project the column away, cast it to a concrete type before
+ persisting, or read the file with Comet disabled for that query.
+
+## Spark 4.2 (Experimental)
+
+Spark 4.2.0-preview4 is provided as experimental support with Java 17 and Scala 2.13.
+
+```{warning}
+Spark 4.2 support is experimental and targets a preview release of Spark. It is intended for early
+evaluation only and should not be used in production.
+```
diff --git a/_sources/user-guide/0.12/configs.md.txt b/_sources/user-guide/0.16/configs.md.txt
similarity index 76%
rename from _sources/user-guide/0.12/configs.md.txt
rename to _sources/user-guide/0.16/configs.md.txt
index 1d0f64587..bb49c351c 100644
--- a/_sources/user-guide/0.12/configs.md.txt
+++ b/_sources/user-guide/0.16/configs.md.txt
@@ -16,9 +16,6 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
# Comet Configuration Settings
@@ -26,56 +23,57 @@ Comet provides the following configuration settings.
## Scan Configuration Settings
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[scan]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
|--------|-------------|---------------|
-| `spark.comet.scan.allowIncompatible` | Some Comet scan implementations are not currently fully compatible with Spark for all datatypes. Set this config to true to allow them anyway. For more information, refer to the [Comet Compatibility Guide](https://datafusion.apache.org/comet/user-guide/compatibility.html). | false |
-| `spark.comet.scan.enabled` | Whether to enable native scans. When this is turned on, Spark will use Comet to read supported data sources (currently only Parquet is supported natively). Note that to enable native vectorized execution, both this config and `spark.comet.exec.enabled` need to be enabled. | true |
-| `spark.comet.scan.icebergNative.enabled` | Whether to enable native Iceberg table scan using iceberg-rust. When enabled, Iceberg tables are read directly through native execution, bypassing Spark's DataSource V2 API for better performance. | false |
-| `spark.comet.scan.preFetch.enabled` | Whether to enable pre-fetching feature of CometScan. | false |
-| `spark.comet.scan.preFetch.threadNum` | The number of threads running pre-fetching for CometScan. Effective if spark.comet.scan.preFetch.enabled is enabled. Note that more pre-fetching threads means more memory requirement to store pre-fetched row groups. | 2 |
+| `spark.comet.scan.icebergNative.dataFileConcurrencyLimit` | The number of Iceberg data files to read concurrently within a single task. Higher values improve throughput for tables with many small files by overlapping I/O latency, but increase memory usage. Values between 2 and 8 are suggested. | 1 |
+| `spark.comet.scan.icebergNative.enabled` | Whether to enable native Iceberg table scan using iceberg-rust. When enabled, Iceberg tables are read directly through native execution, bypassing Spark's DataSource V2 API for better performance. | true |
+| `spark.comet.scan.unsignedSmallIntSafetyCheck` | Parquet files may contain unsigned 8-bit integers (UINT_8) which Spark maps to ShortType. When this config is true (default), Comet falls back to Spark for ShortType columns because we cannot distinguish signed INT16 (safe) from unsigned UINT_8 (may produce different results). Set to false to allow native execution of ShortType columns if you know your data does not contain unsigned UINT_8 columns from improperly encoded Parquet files. F [...]
| `spark.hadoop.fs.comet.libhdfs.schemes` | Defines filesystem schemes (e.g., hdfs, webhdfs) that the native side accesses via libhdfs, separated by commas. Valid only when built with hdfs feature enabled. | |
<!-- prettier-ignore-end -->
<!--END:CONFIG_TABLE-->
## Parquet Reader Configuration Settings
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[parquet]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
|--------|-------------|---------------|
-| `spark.comet.parquet.enable.directBuffer` | Whether to use Java direct byte buffer when reading Parquet. | false |
| `spark.comet.parquet.read.io.adjust.readRange.skew` | In the parallel reader, if the read ranges submitted are skewed in sizes, this option will cause the reader to break up larger read ranges into smaller ranges to reduce the skew. This will result in a slightly larger number of connections opened to the file system but may give improved performance. | false |
| `spark.comet.parquet.read.io.mergeRanges` | When enabled the parallel reader will try to merge ranges of data that are separated by less than `comet.parquet.read.io.mergeRanges.delta` bytes. Longer continuous reads are faster on cloud storage. | true |
| `spark.comet.parquet.read.io.mergeRanges.delta` | The delta in bytes between consecutive read ranges below which the parallel reader will try to merge the ranges. The default is 8MB. | 8388608 |
| `spark.comet.parquet.read.parallel.io.enabled` | Whether to enable Comet's parallel reader for Parquet files. The parallel reader reads ranges of consecutive data in a file in parallel. It is faster for large files and row groups but uses more resources. | true |
| `spark.comet.parquet.read.parallel.io.thread-pool.size` | The maximum number of parallel threads the parallel reader will use in a single executor. For executors configured with a smaller number of cores, use a smaller number. | 16 |
| `spark.comet.parquet.respectFilterPushdown` | Whether to respect Spark's PARQUET_FILTER_PUSHDOWN_ENABLED config. This needs to be respected when running the Spark SQL test suite but the default setting results in poor performance in Comet when using the new native scans, disabled by default | false |
+| `spark.comet.scan.impl` | The implementation of Comet's Parquet scan to use. Available scans are `native_datafusion`, and `native_iceberg_compat`. `native_datafusion` is a fully native implementation, and `native_iceberg_compat` is a hybrid implementation that supports some additional features, such as row indexes and field ids. `auto` (default) chooses the best available scan based on the scan schema. It can be overridden by the environment variable `COMET_PARQUET_SCAN_IMPL`. | auto |
<!-- prettier-ignore-end -->
<!--END:CONFIG_TABLE-->
## Query Execution Settings
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[exec]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
|--------|-------------|---------------|
| `spark.comet.caseConversion.enabled` | Java uses locale-specific rules when converting strings to upper or lower case and Rust does not, so we disable upper and lower by default. | false |
+| `spark.comet.convert.csv.enabled` | When enabled, data from Spark (non-native) CSV v1 and v2 scans will be converted to Arrow format. | false |
+| `spark.comet.convert.json.enabled` | When enabled, data from Spark (non-native) JSON v1 and v2 scans will be converted to Arrow format. | false |
+| `spark.comet.convert.parquet.enabled` | When enabled, data from Spark (non-native) Parquet v1 and v2 scans will be converted to Arrow format. | false |
| `spark.comet.debug.enabled` | Whether to enable debug mode for Comet. When enabled, Comet will do additional checks for debugging purpose. For example, validating array when importing arrays from JVM at native side. Note that these checks may be expensive in performance and should only be enabled for debugging purpose. | false |
-| `spark.comet.dppFallback.enabled` | Whether to fall back to Spark for queries that use DPP. | true |
| `spark.comet.enabled` | Whether to enable Comet extension for Spark. When this is turned on, Spark will use Comet to read Parquet data source. Note that to enable native vectorized execution, both this config and `spark.comet.exec.enabled` need to be enabled. It can be overridden by the environment variable `ENABLE_COMET`. | true |
| `spark.comet.exceptionOnDatetimeRebase` | Whether to throw exception when seeing dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar. Since Spark 3, dates/timestamps were written according to the Proleptic Gregorian calendar. When this is true, Comet will throw exceptions when seeing these dates/timestamps that were written by Spark version before 3.0. If this is false, these dates/timestamps will be read as if they were written to the Proleptic Gregorian calendar and [...]
+| `spark.comet.exec.columnarToRow.native.enabled` | Whether to enable native columnar to row conversion. When enabled, Comet will use native Rust code to convert Arrow columnar data to Spark UnsafeRow format instead of the JVM implementation. This can improve performance for queries that need to convert between columnar and row formats. | true |
| `spark.comet.exec.enabled` | Whether to enable Comet native vectorized execution for Spark. This controls whether Spark should convert operators into their Comet counterparts and execute them in native space. Note: each operator is associated with a separate config in the format of `spark.comet.exec.<operator_name>.enabled` at the moment, and both the config and this need to be turned on, in order for the operator to be executed in native. | true |
| `spark.comet.exec.replaceSortMergeJoin` | Experimental feature to force Spark to replace SortMergeJoin with ShuffledHashJoin for improved performance. This feature is not stable yet. For more information, refer to the [Comet Tuning Guide](https://datafusion.apache.org/comet/user-guide/tuning.html). | false |
| `spark.comet.exec.strictFloatingPoint` | When enabled, fall back to Spark for floating-point operations that may differ from Spark, such as when comparing or sorting -0.0 and 0.0. For more information, refer to the [Comet Compatibility Guide](https://datafusion.apache.org/comet/user-guide/compatibility.html). | false |
| `spark.comet.maxTempDirectorySize` | The maximum amount of data (in bytes) stored inside the temporary directories. | 107374182400b |
+| `spark.comet.metrics.enabled` | Whether to enable Comet metrics reporting through Spark's external monitoring system. When enabled, Comet exposes metrics such as native operators, Spark operators, queries planned, transitions, and acceleration ratio. These metrics can be visualized through tools like Grafana when a metrics sink (e.g., Prometheus) is configured. Disabled by default because Spark plan traversal adds overhead and metrics require a sink to be useful. This config must be se [...]
| `spark.comet.metrics.updateInterval` | The interval in milliseconds to update metrics. If interval is negative, metrics will be updated upon task completion. | 3000 |
| `spark.comet.nativeLoadRequired` | Whether to require Comet native library to load successfully when Comet is enabled. If not, Comet will silently fallback to Spark when it fails to load the native lib. Otherwise, an error will be thrown and the Spark job will be aborted. | false |
-| `spark.comet.regexp.allowIncompatible` | Comet is not currently fully compatible with Spark for all regular expressions. Set this config to true to allow them anyway. For more information, refer to the [Comet Compatibility Guide](https://datafusion.apache.org/comet/user-guide/compatibility.html). | false |
+| `spark.comet.operator.DataWritingCommandExec.allowIncompatible` | Whether to allow incompatibility for operator: DataWritingCommandExec. False by default. Can be overridden with SPARK_COMET_OPERATOR_DATAWRITINGCOMMANDEXEC_ALLOWINCOMPATIBLE env variable It can be overridden by the environment variable `SPARK_COMET_OPERATOR_DATAWRITINGCOMMANDEXEC_ALLOWINCOMPATIBLE`. | false |
+| `spark.comet.sparkToColumnar.enabled` | Whether to enable Spark to Arrow columnar conversion. When this is turned on, Comet will convert operators in `spark.comet.sparkToColumnar.supportedOperatorList` into Arrow columnar format before processing. | false |
+| `spark.comet.sparkToColumnar.supportedOperatorList` | A comma-separated list of operators that will be converted to Arrow columnar format when `spark.comet.sparkToColumnar.enabled` is true. | Range,InMemoryTableScan,RDDScan,OneRowRelation |
<!-- prettier-ignore-end -->
<!--END:CONFIG_TABLE-->
@@ -83,7 +81,6 @@ Comet provides the following configuration settings.
These settings can be used to determine which parts of the plan are accelerated by Comet and to see why some parts of the plan could not be supported by Comet.
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[exec_explain]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
@@ -98,7 +95,6 @@ These settings can be used to determine which parts of the plan are accelerated
## Shuffle Configuration Settings
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[shuffle]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
@@ -109,9 +105,15 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.columnar.shuffle.batch.size` | Batch size when writing out sorted spill files on the native side. Note that this should not be larger than batch size (i.e., `spark.comet.batchSize`). Otherwise it will produce larger batches than expected in the native operator after shuffle. | 8192 |
| `spark.comet.exec.shuffle.compression.codec` | The codec of Comet native shuffle used to compress shuffle data. lz4, zstd, and snappy are supported. Compression can be disabled by setting spark.shuffle.compress=false. | lz4 |
| `spark.comet.exec.shuffle.compression.zstd.level` | The compression level to use when compressing shuffle files with zstd. | 1 |
+| `spark.comet.exec.shuffle.convertFromSparkPlan.enabled` | When enabled, Comet will convert a Spark `ShuffleExchangeExec` to a Comet columnar shuffle even when its child is a non-Comet (Spark) plan. Disable to leave such shuffles as native Spark shuffles, restricting Comet shuffle to cases where the child is already a Comet plan. | true |
+| `spark.comet.exec.shuffle.directRead.enabled` | When enabled, native operators that consume shuffle output will read compressed shuffle blocks directly in native code, bypassing Arrow FFI. Applies to both native shuffle and JVM columnar shuffle. Requires spark.comet.exec.shuffle.enabled to be true. | true |
| `spark.comet.exec.shuffle.enabled` | Whether to enable Comet native shuffle. Note that this requires setting `spark.shuffle.manager` to `org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager`. `spark.shuffle.manager` must be set before starting the Spark application and cannot be changed during the application. | true |
+| `spark.comet.exec.shuffle.revertRedundantColumnar.enabled` | When enabled, Comet reverts a `CometShuffleExchangeExec` with `CometColumnarShuffle` back to Spark's `ShuffleExchangeExec` when both its parent and child are non-Comet hash aggregate operators. This avoids a redundant row -> Arrow -> shuffle -> Arrow -> row conversion when no Comet operator on either side can consume columnar output. Disable to keep Comet columnar shuffle even in that case, which preserves Comet's off-heap sh [...]
+| `spark.comet.exec.shuffle.writeBufferSize` | Size of the write buffer in bytes used by the native shuffle writer when writing shuffle data to disk. Larger values may improve write performance by reducing the number of system calls, but will use more memory. The default is 1MB which provides a good balance between performance and memory usage. | 1048576b |
| `spark.comet.native.shuffle.partitioning.hash.enabled` | Whether to enable hash partitioning for Comet native shuffle. | true |
| `spark.comet.native.shuffle.partitioning.range.enabled` | Whether to enable range partitioning for Comet native shuffle. | true |
+| `spark.comet.native.shuffle.partitioning.roundrobin.enabled` | Whether to enable round robin partitioning for Comet native shuffle. This is disabled by default because Comet's round-robin produces different partition assignments than Spark. Spark sorts rows by their binary UnsafeRow representation before assigning partitions, but Comet uses Arrow format which has a different binary layout. Instead, Comet implements round-robin as hash partitioning on all columns, which achieves the sam [...]
+| `spark.comet.native.shuffle.partitioning.roundrobin.maxHashColumns` | The maximum number of columns to hash for round robin partitioning. When set to 0 (the default), all columns are hashed. When set to a positive value, only the first N columns are used for hashing, which can improve performance for wide tables while still providing reasonable distribution. | 0 |
| `spark.comet.shuffle.preferDictionary.ratio` | The ratio of total values to distinct values in a string column to decide whether to prefer dictionary encoding when shuffling the column. If the ratio is higher than this config, dictionary encoding will be used on shuffling string column. This config is effective if it is higher than 1.0. Note that this config is only used when `spark.comet.exec.shuffle.mode` is `jvm`. | 10.0 |
| `spark.comet.shuffle.sizeInBytesMultiplier` | Comet reports smaller sizes for shuffle due to using Arrow's columnar memory format and this can result in Spark choosing a different join strategy due to the estimated size of the exchange being smaller. Comet will multiple sizeInBytes by this amount to avoid regressions in join strategy. | 1.0 |
<!-- prettier-ignore-end -->
@@ -119,7 +121,6 @@ These settings can be used to determine which parts of the plan are accelerated
## Memory & Tuning Configuration Settings
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[tuning]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
@@ -127,33 +128,31 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.batchSize` | The columnar batch size, i.e., the maximum number of rows that a batch can contain. | 8192 |
| `spark.comet.exec.memoryPool` | The type of memory pool to be used for Comet native execution when running Spark in off-heap mode. Available pool types are `greedy_unified` and `fair_unified`. For more information, refer to the [Comet Tuning Guide](https://datafusion.apache.org/comet/user-guide/tuning.html). | fair_unified |
| `spark.comet.exec.memoryPool.fraction` | Fraction of off-heap memory pool that is available to Comet. Only applies to off-heap mode. For more information, refer to the [Comet Tuning Guide](https://datafusion.apache.org/comet/user-guide/tuning.html). | 1.0 |
-| `spark.comet.tracing.enabled` | Enable fine-grained tracing of events and memory usage. For more information, refer to the [Comet Tracing Guide](https://datafusion.apache.org/comet/user-guide/tracing.html). | false |
+| `spark.comet.tracing.enabled` | Enable fine-grained tracing of events and memory usage. For more information, refer to the [Comet Tracing Guide](https://datafusion.apache.org/comet/contributor-guide/tracing.html). | false |
<!-- prettier-ignore-end -->
<!--END:CONFIG_TABLE-->
## Development & Testing Settings
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[testing]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
|--------|-------------|---------------|
| `spark.comet.columnar.shuffle.memory.factor` | Fraction of Comet memory to be allocated per executor process for columnar shuffle when running in on-heap mode. For more information, refer to the [Comet Tuning Guide](https://datafusion.apache.org/comet/user-guide/tuning.html). | 1.0 |
-| `spark.comet.convert.csv.enabled` | When enabled, data from Spark (non-native) CSV v1 and v2 scans will be converted to Arrow format. This is an experimental feature and has known issues with non-UTC timezones. | false |
-| `spark.comet.convert.json.enabled` | When enabled, data from Spark (non-native) JSON v1 and v2 scans will be converted to Arrow format. This is an experimental feature and has known issues with non-UTC timezones. | false |
-| `spark.comet.convert.parquet.enabled` | When enabled, data from Spark (non-native) Parquet v1 and v2 scans will be converted to Arrow format. This is an experimental feature and has known issues with non-UTC timezones. | false |
+| `spark.comet.debug.memory` | When enabled, log all native memory pool interactions. For more information, refer to the Comet Debugging Guide (https://datafusion.apache.org/comet/contributor-guide/debugging.html). | false |
| `spark.comet.exec.onHeap.enabled` | Whether to allow Comet to run in on-heap mode. Required for running Spark SQL tests. It can be overridden by the environment variable `ENABLE_COMET_ONHEAP`. | false |
| `spark.comet.exec.onHeap.memoryPool` | The type of memory pool to be used for Comet native execution when running Spark in on-heap mode. Available pool types are `greedy`, `fair_spill`, `greedy_task_shared`, `fair_spill_task_shared`, `greedy_global`, `fair_spill_global`, and `unbounded`. | greedy_task_shared |
+| `spark.comet.exec.respectDataFusionConfigs` | Development and testing configuration option to allow DataFusion configs set in Spark configuration settings starting with `spark.comet.datafusion.` to be passed into native execution. | false |
| `spark.comet.memoryOverhead` | The amount of additional memory to be allocated per executor process for Comet, in MiB, when running Spark in on-heap mode. | 1024 MiB |
-| `spark.comet.sparkToColumnar.enabled` | Whether to enable Spark to Arrow columnar conversion. When this is turned on, Comet will convert operators in `spark.comet.sparkToColumnar.supportedOperatorList` into Arrow columnar format before processing. This is an experimental feature and has known issues with non-UTC timezones. | false |
-| `spark.comet.sparkToColumnar.supportedOperatorList` | A comma-separated list of operators that will be converted to Arrow columnar format when `spark.comet.sparkToColumnar.enabled` is true. | Range,InMemoryTableScan,RDDScan |
+| `spark.comet.parquet.write.enabled` | Whether to enable native Parquet write through Comet. When enabled, Comet will intercept Parquet write operations and execute them natively. This feature is highly experimental and only partially implemented. It should not be used in production. It can be overridden by the environment variable `ENABLE_COMET_WRITE`. | false |
+| `spark.comet.scan.csv.v2.enabled` | Whether to use the native Comet V2 CSV reader for improved performance. Default: false (uses standard Spark CSV reader) Experimental: Performance benefits are workload-dependent. | false |
+| `spark.comet.scan.enabled` | Whether to enable native scans. Intended for use in Comet's own test suites to selectively disable native scans; not intended for production use. | true |
| `spark.comet.testing.strict` | Experimental option to enable strict testing, which will fail tests that could be more comprehensive, such as checking for a specific fallback reason. It can be overridden by the environment variable `ENABLE_COMET_STRICT_TESTING`. | false |
<!-- prettier-ignore-end -->
<!--END:CONFIG_TABLE-->
## Enabling or Disabling Individual Operators
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[enable_exec]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
@@ -164,6 +163,7 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.exec.coalesce.enabled` | Whether to enable coalesce by default. | true |
| `spark.comet.exec.collectLimit.enabled` | Whether to enable collectLimit by default. | true |
| `spark.comet.exec.expand.enabled` | Whether to enable expand by default. | true |
+| `spark.comet.exec.explode.enabled` | Whether to enable explode by default. | true |
| `spark.comet.exec.filter.enabled` | Whether to enable filter by default. | true |
| `spark.comet.exec.globalLimit.enabled` | Whether to enable globalLimit by default. | true |
| `spark.comet.exec.hashJoin.enabled` | Whether to enable hashJoin by default. | true |
@@ -181,13 +181,13 @@ These settings can be used to determine which parts of the plan are accelerated
## Enabling or Disabling Individual Scalar Expressions
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[enable_expr]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
|--------|-------------|---------------|
| `spark.comet.expression.Abs.enabled` | Enable Comet acceleration for `Abs` | true |
| `spark.comet.expression.Acos.enabled` | Enable Comet acceleration for `Acos` | true |
+| `spark.comet.expression.Acosh.enabled` | Enable Comet acceleration for `Acosh` | true |
| `spark.comet.expression.Add.enabled` | Enable Comet acceleration for `Add` | true |
| `spark.comet.expression.Alias.enabled` | Enable Comet acceleration for `Alias` | true |
| `spark.comet.expression.And.enabled` | Enable Comet acceleration for `And` | true |
@@ -202,15 +202,20 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.ArrayJoin.enabled` | Enable Comet acceleration for `ArrayJoin` | true |
| `spark.comet.expression.ArrayMax.enabled` | Enable Comet acceleration for `ArrayMax` | true |
| `spark.comet.expression.ArrayMin.enabled` | Enable Comet acceleration for `ArrayMin` | true |
+| `spark.comet.expression.ArrayPosition.enabled` | Enable Comet acceleration for `ArrayPosition` | true |
| `spark.comet.expression.ArrayRemove.enabled` | Enable Comet acceleration for `ArrayRemove` | true |
| `spark.comet.expression.ArrayRepeat.enabled` | Enable Comet acceleration for `ArrayRepeat` | true |
| `spark.comet.expression.ArrayUnion.enabled` | Enable Comet acceleration for `ArrayUnion` | true |
| `spark.comet.expression.ArraysOverlap.enabled` | Enable Comet acceleration for `ArraysOverlap` | true |
+| `spark.comet.expression.ArraysZip.enabled` | Enable Comet acceleration for `ArraysZip` | true |
| `spark.comet.expression.Ascii.enabled` | Enable Comet acceleration for `Ascii` | true |
| `spark.comet.expression.Asin.enabled` | Enable Comet acceleration for `Asin` | true |
+| `spark.comet.expression.Asinh.enabled` | Enable Comet acceleration for `Asinh` | true |
| `spark.comet.expression.Atan.enabled` | Enable Comet acceleration for `Atan` | true |
| `spark.comet.expression.Atan2.enabled` | Enable Comet acceleration for `Atan2` | true |
+| `spark.comet.expression.Atanh.enabled` | Enable Comet acceleration for `Atanh` | true |
| `spark.comet.expression.AttributeReference.enabled` | Enable Comet acceleration for `AttributeReference` | true |
+| `spark.comet.expression.Bin.enabled` | Enable Comet acceleration for `Bin` | true |
| `spark.comet.expression.BitLength.enabled` | Enable Comet acceleration for `BitLength` | true |
| `spark.comet.expression.BitwiseAnd.enabled` | Enable Comet acceleration for `BitwiseAnd` | true |
| `spark.comet.expression.BitwiseCount.enabled` | Enable Comet acceleration for `BitwiseCount` | true |
@@ -221,6 +226,7 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.BloomFilterMightContain.enabled` | Enable Comet acceleration for `BloomFilterMightContain` | true |
| `spark.comet.expression.CaseWhen.enabled` | Enable Comet acceleration for `CaseWhen` | true |
| `spark.comet.expression.Cast.enabled` | Enable Comet acceleration for `Cast` | true |
+| `spark.comet.expression.Cbrt.enabled` | Enable Comet acceleration for `Cbrt` | true |
| `spark.comet.expression.Ceil.enabled` | Enable Comet acceleration for `Ceil` | true |
| `spark.comet.expression.CheckOverflow.enabled` | Enable Comet acceleration for `CheckOverflow` | true |
| `spark.comet.expression.Chr.enabled` | Enable Comet acceleration for `Chr` | true |
@@ -231,13 +237,18 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.Cos.enabled` | Enable Comet acceleration for `Cos` | true |
| `spark.comet.expression.Cosh.enabled` | Enable Comet acceleration for `Cosh` | true |
| `spark.comet.expression.Cot.enabled` | Enable Comet acceleration for `Cot` | true |
+| `spark.comet.expression.Crc32.enabled` | Enable Comet acceleration for `Crc32` | true |
| `spark.comet.expression.CreateArray.enabled` | Enable Comet acceleration for `CreateArray` | true |
| `spark.comet.expression.CreateNamedStruct.enabled` | Enable Comet acceleration for `CreateNamedStruct` | true |
| `spark.comet.expression.DateAdd.enabled` | Enable Comet acceleration for `DateAdd` | true |
+| `spark.comet.expression.DateDiff.enabled` | Enable Comet acceleration for `DateDiff` | true |
+| `spark.comet.expression.DateFormatClass.enabled` | Enable Comet acceleration for `DateFormatClass` | true |
+| `spark.comet.expression.DateFromUnixDate.enabled` | Enable Comet acceleration for `DateFromUnixDate` | true |
| `spark.comet.expression.DateSub.enabled` | Enable Comet acceleration for `DateSub` | true |
| `spark.comet.expression.DayOfMonth.enabled` | Enable Comet acceleration for `DayOfMonth` | true |
| `spark.comet.expression.DayOfWeek.enabled` | Enable Comet acceleration for `DayOfWeek` | true |
| `spark.comet.expression.DayOfYear.enabled` | Enable Comet acceleration for `DayOfYear` | true |
+| `spark.comet.expression.Days.enabled` | Enable Comet acceleration for `Days` | true |
| `spark.comet.expression.Divide.enabled` | Enable Comet acceleration for `Divide` | true |
| `spark.comet.expression.ElementAt.enabled` | Enable Comet acceleration for `ElementAt` | true |
| `spark.comet.expression.EndsWith.enabled` | Enable Comet acceleration for `EndsWith` | true |
@@ -250,12 +261,14 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.FromUnixTime.enabled` | Enable Comet acceleration for `FromUnixTime` | true |
| `spark.comet.expression.GetArrayItem.enabled` | Enable Comet acceleration for `GetArrayItem` | true |
| `spark.comet.expression.GetArrayStructFields.enabled` | Enable Comet acceleration for `GetArrayStructFields` | true |
+| `spark.comet.expression.GetJsonObject.enabled` | Enable Comet acceleration for `GetJsonObject` | true |
| `spark.comet.expression.GetMapValue.enabled` | Enable Comet acceleration for `GetMapValue` | true |
| `spark.comet.expression.GetStructField.enabled` | Enable Comet acceleration for `GetStructField` | true |
| `spark.comet.expression.GreaterThan.enabled` | Enable Comet acceleration for `GreaterThan` | true |
| `spark.comet.expression.GreaterThanOrEqual.enabled` | Enable Comet acceleration for `GreaterThanOrEqual` | true |
| `spark.comet.expression.Hex.enabled` | Enable Comet acceleration for `Hex` | true |
| `spark.comet.expression.Hour.enabled` | Enable Comet acceleration for `Hour` | true |
+| `spark.comet.expression.Hours.enabled` | Enable Comet acceleration for `Hours` | true |
| `spark.comet.expression.If.enabled` | Enable Comet acceleration for `If` | true |
| `spark.comet.expression.In.enabled` | Enable Comet acceleration for `In` | true |
| `spark.comet.expression.InSet.enabled` | Enable Comet acceleration for `InSet` | true |
@@ -264,7 +277,10 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.IsNaN.enabled` | Enable Comet acceleration for `IsNaN` | true |
| `spark.comet.expression.IsNotNull.enabled` | Enable Comet acceleration for `IsNotNull` | true |
| `spark.comet.expression.IsNull.enabled` | Enable Comet acceleration for `IsNull` | true |
+| `spark.comet.expression.JsonToStructs.enabled` | Enable Comet acceleration for `JsonToStructs` | true |
| `spark.comet.expression.KnownFloatingPointNormalized.enabled` | Enable Comet acceleration for `KnownFloatingPointNormalized` | true |
+| `spark.comet.expression.LastDay.enabled` | Enable Comet acceleration for `LastDay` | true |
+| `spark.comet.expression.Left.enabled` | Enable Comet acceleration for `Left` | true |
| `spark.comet.expression.Length.enabled` | Enable Comet acceleration for `Length` | true |
| `spark.comet.expression.LessThan.enabled` | Enable Comet acceleration for `LessThan` | true |
| `spark.comet.expression.LessThanOrEqual.enabled` | Enable Comet acceleration for `LessThanOrEqual` | true |
@@ -273,10 +289,14 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.Log.enabled` | Enable Comet acceleration for `Log` | true |
| `spark.comet.expression.Log10.enabled` | Enable Comet acceleration for `Log10` | true |
| `spark.comet.expression.Log2.enabled` | Enable Comet acceleration for `Log2` | true |
+| `spark.comet.expression.Logarithm.enabled` | Enable Comet acceleration for `Logarithm` | true |
| `spark.comet.expression.Lower.enabled` | Enable Comet acceleration for `Lower` | true |
+| `spark.comet.expression.MakeDate.enabled` | Enable Comet acceleration for `MakeDate` | true |
| `spark.comet.expression.MakeDecimal.enabled` | Enable Comet acceleration for `MakeDecimal` | true |
+| `spark.comet.expression.MapContainsKey.enabled` | Enable Comet acceleration for `MapContainsKey` | true |
| `spark.comet.expression.MapEntries.enabled` | Enable Comet acceleration for `MapEntries` | true |
| `spark.comet.expression.MapFromArrays.enabled` | Enable Comet acceleration for `MapFromArrays` | true |
+| `spark.comet.expression.MapFromEntries.enabled` | Enable Comet acceleration for `MapFromEntries` | true |
| `spark.comet.expression.MapKeys.enabled` | Enable Comet acceleration for `MapKeys` | true |
| `spark.comet.expression.MapValues.enabled` | Enable Comet acceleration for `MapValues` | true |
| `spark.comet.expression.Md5.enabled` | Enable Comet acceleration for `Md5` | true |
@@ -285,9 +305,11 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.Month.enabled` | Enable Comet acceleration for `Month` | true |
| `spark.comet.expression.Multiply.enabled` | Enable Comet acceleration for `Multiply` | true |
| `spark.comet.expression.Murmur3Hash.enabled` | Enable Comet acceleration for `Murmur3Hash` | true |
+| `spark.comet.expression.NextDay.enabled` | Enable Comet acceleration for `NextDay` | true |
| `spark.comet.expression.Not.enabled` | Enable Comet acceleration for `Not` | true |
| `spark.comet.expression.OctetLength.enabled` | Enable Comet acceleration for `OctetLength` | true |
| `spark.comet.expression.Or.enabled` | Enable Comet acceleration for `Or` | true |
+| `spark.comet.expression.Pi.enabled` | Enable Comet acceleration for `Pi` | true |
| `spark.comet.expression.Pow.enabled` | Enable Comet acceleration for `Pow` | true |
| `spark.comet.expression.Quarter.enabled` | Enable Comet acceleration for `Quarter` | true |
| `spark.comet.expression.RLike.enabled` | Enable Comet acceleration for `RLike` | true |
@@ -296,9 +318,11 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.RegExpReplace.enabled` | Enable Comet acceleration for `RegExpReplace` | true |
| `spark.comet.expression.Remainder.enabled` | Enable Comet acceleration for `Remainder` | true |
| `spark.comet.expression.Reverse.enabled` | Enable Comet acceleration for `Reverse` | true |
+| `spark.comet.expression.Right.enabled` | Enable Comet acceleration for `Right` | true |
| `spark.comet.expression.Round.enabled` | Enable Comet acceleration for `Round` | true |
| `spark.comet.expression.ScalarSubquery.enabled` | Enable Comet acceleration for `ScalarSubquery` | true |
| `spark.comet.expression.Second.enabled` | Enable Comet acceleration for `Second` | true |
+| `spark.comet.expression.SecondsToTimestamp.enabled` | Enable Comet acceleration for `SecondsToTimestamp` | true |
| `spark.comet.expression.Sha1.enabled` | Enable Comet acceleration for `Sha1` | true |
| `spark.comet.expression.Sha2.enabled` | Enable Comet acceleration for `Sha2` | true |
| `spark.comet.expression.ShiftLeft.enabled` | Enable Comet acceleration for `ShiftLeft` | true |
@@ -306,6 +330,8 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.Signum.enabled` | Enable Comet acceleration for `Signum` | true |
| `spark.comet.expression.Sin.enabled` | Enable Comet acceleration for `Sin` | true |
| `spark.comet.expression.Sinh.enabled` | Enable Comet acceleration for `Sinh` | true |
+| `spark.comet.expression.Size.enabled` | Enable Comet acceleration for `Size` | true |
+| `spark.comet.expression.SortArray.enabled` | Enable Comet acceleration for `SortArray` | true |
| `spark.comet.expression.SortOrder.enabled` | Enable Comet acceleration for `SortOrder` | true |
| `spark.comet.expression.SparkPartitionID.enabled` | Enable Comet acceleration for `SparkPartitionID` | true |
| `spark.comet.expression.Sqrt.enabled` | Enable Comet acceleration for `Sqrt` | true |
@@ -317,20 +343,27 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.StringRepeat.enabled` | Enable Comet acceleration for `StringRepeat` | true |
| `spark.comet.expression.StringReplace.enabled` | Enable Comet acceleration for `StringReplace` | true |
| `spark.comet.expression.StringSpace.enabled` | Enable Comet acceleration for `StringSpace` | true |
+| `spark.comet.expression.StringSplit.enabled` | Enable Comet acceleration for `StringSplit` | true |
+| `spark.comet.expression.StringToMap.enabled` | Enable Comet acceleration for `StringToMap` | true |
| `spark.comet.expression.StringTranslate.enabled` | Enable Comet acceleration for `StringTranslate` | true |
| `spark.comet.expression.StringTrim.enabled` | Enable Comet acceleration for `StringTrim` | true |
| `spark.comet.expression.StringTrimBoth.enabled` | Enable Comet acceleration for `StringTrimBoth` | true |
| `spark.comet.expression.StringTrimLeft.enabled` | Enable Comet acceleration for `StringTrimLeft` | true |
| `spark.comet.expression.StringTrimRight.enabled` | Enable Comet acceleration for `StringTrimRight` | true |
+| `spark.comet.expression.StructsToCsv.enabled` | Enable Comet acceleration for `StructsToCsv` | true |
| `spark.comet.expression.StructsToJson.enabled` | Enable Comet acceleration for `StructsToJson` | true |
| `spark.comet.expression.Substring.enabled` | Enable Comet acceleration for `Substring` | true |
| `spark.comet.expression.Subtract.enabled` | Enable Comet acceleration for `Subtract` | true |
| `spark.comet.expression.Tan.enabled` | Enable Comet acceleration for `Tan` | true |
| `spark.comet.expression.Tanh.enabled` | Enable Comet acceleration for `Tanh` | true |
+| `spark.comet.expression.ToDegrees.enabled` | Enable Comet acceleration for `ToDegrees` | true |
+| `spark.comet.expression.ToRadians.enabled` | Enable Comet acceleration for `ToRadians` | true |
| `spark.comet.expression.TruncDate.enabled` | Enable Comet acceleration for `TruncDate` | true |
| `spark.comet.expression.TruncTimestamp.enabled` | Enable Comet acceleration for `TruncTimestamp` | true |
| `spark.comet.expression.UnaryMinus.enabled` | Enable Comet acceleration for `UnaryMinus` | true |
| `spark.comet.expression.Unhex.enabled` | Enable Comet acceleration for `Unhex` | true |
+| `spark.comet.expression.UnixDate.enabled` | Enable Comet acceleration for `UnixDate` | true |
+| `spark.comet.expression.UnixTimestamp.enabled` | Enable Comet acceleration for `UnixTimestamp` | true |
| `spark.comet.expression.UnscaledValue.enabled` | Enable Comet acceleration for `UnscaledValue` | true |
| `spark.comet.expression.Upper.enabled` | Enable Comet acceleration for `Upper` | true |
| `spark.comet.expression.WeekDay.enabled` | Enable Comet acceleration for `WeekDay` | true |
@@ -342,7 +375,6 @@ These settings can be used to determine which parts of the plan are accelerated
## Enabling or Disabling Individual Aggregate Expressions
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
<!--BEGIN:CONFIG_TABLE[enable_agg_expr]-->
<!-- prettier-ignore-start -->
| Config | Description | Default Value |
@@ -352,6 +384,7 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.BitOrAgg.enabled` | Enable Comet acceleration for `BitOrAgg` | true |
| `spark.comet.expression.BitXorAgg.enabled` | Enable Comet acceleration for `BitXorAgg` | true |
| `spark.comet.expression.BloomFilterAggregate.enabled` | Enable Comet acceleration for `BloomFilterAggregate` | true |
+| `spark.comet.expression.CollectSet.enabled` | Enable Comet acceleration for `CollectSet` | true |
| `spark.comet.expression.Corr.enabled` | Enable Comet acceleration for `Corr` | true |
| `spark.comet.expression.Count.enabled` | Enable Comet acceleration for `Count` | true |
| `spark.comet.expression.CovPopulation.enabled` | Enable Comet acceleration for `CovPopulation` | true |
diff --git a/_sources/user-guide/0.12/datasources.md.txt b/_sources/user-guide/0.16/datasources.md.txt
similarity index 65%
rename from _sources/user-guide/0.12/datasources.md.txt
rename to _sources/user-guide/0.16/datasources.md.txt
index 494a9eb21..0e1708767 100644
--- a/_sources/user-guide/0.12/datasources.md.txt
+++ b/_sources/user-guide/0.16/datasources.md.txt
@@ -16,9 +16,6 @@
specific language governing permissions and limitations
under the License.
-->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
# Supported Spark Data Sources
@@ -26,10 +23,9 @@ This is **out-of-date** documentation. The latest Comet release is version 0.15.
### Parquet
-When `spark.comet.scan.enabled` is enabled, Parquet scans will be performed natively by Comet if all data types
-in the schema are supported. When this option is not enabled, the scan will fall back to Spark. In this case,
-enabling `spark.comet.convert.parquet.enabled` will immediately convert the data into Arrow format, allowing native
-execution to happen after that, but the process may not be efficient.
+Parquet scans are performed natively by Comet if all data types in the schema are supported. When the scan
+falls back to Spark, enabling `spark.comet.convert.parquet.enabled` will immediately convert the data into
+Arrow format, allowing native execution to happen after that, but the process may not be efficient.
### Apache Iceberg
@@ -39,7 +35,11 @@ Comet accelerates Iceberg scans of Parquet files. See the [Iceberg Guide] for mo
### CSV
-Comet does not provide native CSV scan, but when `spark.comet.convert.csv.enabled` is enabled, data is immediately
+Comet provides experimental native CSV scan support. When `spark.comet.scan.csv.v2.enabled` is enabled, CSV files
+are read natively for improved performance. This feature is experimental and performance benefits are
+workload-dependent.
+
+Alternatively, when `spark.comet.convert.csv.enabled` is enabled, data from Spark's CSV reader is immediately
converted into Arrow format, allowing native execution to happen after that.
### JSON
@@ -68,12 +68,12 @@ Unlike to native Comet reader the Datafusion reader fully supports nested types
To build Comet with native DataFusion reader and remote HDFS support it is required to have a JDK installed
Example:
-Build a Comet for `spark-3.5` provide a JDK path in `JAVA_HOME`
+Build a Comet for `spark-4.1` provide a JDK path in `JAVA_HOME`
Provide the JRE linker path in `RUSTFLAGS`, the path can vary depending on the system. Typically JRE linker is a part of installed JDK
```shell
-export JAVA_HOME="/opt/homebrew/opt/openjdk@11"
-make release PROFILES="-Pspark-3.5" COMET_FEATURES=hdfs RUSTFLAGS="-L $JAVA_HOME/libexec/openjdk.jdk/Contents/Home/lib/server"
+export JAVA_HOME="/opt/homebrew/opt/openjdk@17"
+make release PROFILES="-Pspark-4.1" COMET_FEATURES=hdfs RUSTFLAGS="-L $JAVA_HOME/libexec/openjdk.jdk/Contents/Home/lib/server"
```
Start Comet with experimental reader and HDFS support as [described](installation.md/#run-spark-shell-with-comet-enabled)
@@ -99,7 +99,7 @@ root
| |-- lastName: string (nullable = true)
| |-- ageInYears: integer (nullable = true)
-25/01/30 16:50:43 INFO core/src/lib.rs: Comet native library version 0.12.0 initialized
+25/01/30 16:50:43 INFO core/src/lib.rs: Comet native library version 0.16.0 initialized
== Physical Plan ==
* CometColumnarToRow (2)
+- CometNativeScan: (1)
@@ -148,7 +148,7 @@ docker compose -f kube/local/hdfs-docker-compose.yml up
- Build a project with HDFS support
```shell
-JAVA_HOME="/opt/homebrew/opt/openjdk@11" make release PROFILES="-Pspark-3.5" COMET_FEATURES=hdfs RUSTFLAGS="-L /opt/homebrew/opt/openjdk@11/libexec/openjdk.jdk/Contents/Home/lib/server"
+JAVA_HOME="/opt/homebrew/opt/openjdk@17" make release PROFILES="-Pspark-4.1" COMET_FEATURES=hdfs RUSTFLAGS="-L /opt/homebrew/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home/lib/server"
```
- Run local test
@@ -173,7 +173,15 @@ Or use `spark-shell` with HDFS support as described [above](#using-experimental-
## S3
-#### Root CA Certificates
+The `native_datafusion` and `native_iceberg_compat` Parquet scan implementations completely offload data loading
+to native code. They use the [`object_store` crate](https://crates.io/crates/object_store) to read data from S3 and
+support configuring S3 access using standard [Hadoop S3A configurations](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html#General_S3A_Client_configuration) by translating them to
+the `object_store` crate's format.
+
+This implementation maintains compatibility with existing Hadoop S3A configurations, so existing code will
+continue to work as long as the configurations are supported and can be translated without loss of functionality.
+
+### Root CA Certificates
One major difference between Spark and Comet is the mechanism for discovering Root
CA Certificates. Spark uses the JVM to read CA Certificates from the Java Trust Store, but native Comet
@@ -181,7 +189,7 @@ scans use system Root CA Certificates (typically stored
in `/etc/ssl/certs` on Linux). These scans will not be able to interact with S3 if the Root CA Certificates are not
installed.
-#### Supported Credential Providers
+### Supported Credential Providers
AWS credential providers can be configured using the `fs.s3a.aws.credentials.provider` configuration. The following table shows the supported credential providers and their configuration options:
@@ -196,5 +204,61 @@ AWS credential providers can be configured using the `fs.s3a.aws.credentials.pro
| `com.amazonaws.auth.InstanceProfileCredentialsProvider`<br/>`software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider` | Access S3 using EC2 instance metadata service (IMDS) | None |
| `com.amazonaws.auth.ContainerCredentialsProvider`<br/>`software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider`<br/>`com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper` | Access S3 using ECS task credentials | None |
| `com.amazonaws.auth.WebIdentityTokenCredentialsProvider`<br/>`software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider` | Authenticate using web identity token file | None |
+| `com.amazonaws.auth.profile.ProfileCredentialsProvider`<br/>`software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider` | Authenticate using a named profile from the local AWS credentials file | None |
Multiple credential providers can be specified in a comma-separated list using the `fs.s3a.aws.credentials.provider` configuration, just as Hadoop AWS supports. If `fs.s3a.aws.credentials.provider` is not configured, Hadoop S3A's default credential provider chain will be used. All configuration options also support bucket-specific overrides using the pattern `fs.s3a.bucket.{bucket-name}.{option}`.
+
+### Additional S3 Configuration Options
+
+Beyond credential providers, the `native_datafusion` and `native_iceberg_compat` implementations support additional
+S3 configuration options:
+
+| Option | Description |
+| ------------------------------- | -------------------------------------------------------------------------------------------------- |
+| `fs.s3a.endpoint` | The endpoint of the S3 service |
+| `fs.s3a.endpoint.region` | The AWS region for the S3 service. If not specified, the region will be auto-detected. |
+| `fs.s3a.path.style.access` | Whether to use path style access for the S3 service (true/false, defaults to virtual hosted style) |
+| `fs.s3a.requester.pays.enabled` | Whether to enable requester pays for S3 requests (true/false) |
+
+All configuration options support bucket-specific overrides using the pattern `fs.s3a.bucket.{bucket-name}.{option}`.
+
+### Examples
+
+The following examples demonstrate how to configure S3 access with the `native_datafusion` and `native_iceberg_compat`
+Parquet scan implementations using different authentication methods.
+
+**Example 1: Simple Credentials**
+
+This example shows how to access a private S3 bucket using an access key and secret key. The `fs.s3a.aws.credentials.provider` configuration can be omitted since `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider` is included in Hadoop S3A's default credential provider chain.
+
+```shell
+$SPARK_HOME/bin/spark-shell \
+...
+--conf spark.comet.scan.impl=native_datafusion \
+--conf spark.hadoop.fs.s3a.access.key=my-access-key \
+--conf spark.hadoop.fs.s3a.secret.key=my-secret-key
+...
+```
+
+**Example 2: Assume Role with Web Identity Token**
+
+This example demonstrates using an assumed role credential to access a private S3 bucket, where the base credential for assuming the role is provided by a web identity token credentials provider.
+
+```shell
+$SPARK_HOME/bin/spark-shell \
+...
+--conf spark.comet.scan.impl=native_datafusion \
+--conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider \
+--conf spark.hadoop.fs.s3a.assumed.role.arn=arn:aws:iam::123456789012:role/my-role \
+--conf spark.hadoop.fs.s3a.assumed.role.session.name=my-session \
+--conf spark.hadoop.fs.s3a.assumed.role.credentials.provider=com.amazonaws.auth.WebIdentityTokenCredentialsProvider
+...
+```
+
+### Limitations
+
+The S3 support of `native_datafusion` and `native_iceberg_compat` has the following limitations:
+
+1. **Partial Hadoop S3A configuration support**: Not all Hadoop S3A configurations are currently supported. Only the configurations listed in the tables above are translated and applied to the underlying `object_store` crate.
+
+2. **Custom credential providers**: Custom implementations of AWS credential providers are not supported. The implementation only supports the standard credential providers listed in the table above. We are planning to add support for custom credential providers through a JNI-based adapter that will allow calling Java credential providers from native code. See [issue #1829](https://github.com/apache/datafusion-comet/issues/1829) for more details.
diff --git a/_sources/user-guide/0.12/datatypes.md.txt b/_sources/user-guide/0.16/datatypes.md.txt
similarity index 90%
rename from _sources/user-guide/0.12/datatypes.md.txt
rename to _sources/user-guide/0.16/datatypes.md.txt
index e68dd9e84..7bc4d6616 100644
--- a/_sources/user-guide/0.12/datatypes.md.txt
+++ b/_sources/user-guide/0.16/datatypes.md.txt
@@ -16,16 +16,13 @@
specific language governing permissions and limitations
under the License.
-->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
# Supported Spark Data Types
Comet supports the following Spark data types. Refer to the [Comet Compatibility Guide] for information about data
type support in scans and other operators.
-[Comet Compatibility Guide]: compatibility.md
+[Comet Compatibility Guide]: compatibility/index.md
<!-- based on org.apache.comet.serde.QueryPlanSerde.supportedDataType -->
diff --git a/_sources/user-guide/0.16/expressions.md.txt b/_sources/user-guide/0.16/expressions.md.txt
new file mode 100644
index 000000000..3842148a4
--- /dev/null
+++ b/_sources/user-guide/0.16/expressions.md.txt
@@ -0,0 +1,316 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Supported Spark Expressions
+
+Comet supports the following Spark expressions. See the [Comet Compatibility Guide] for details on known
+incompatibilities and unsupported cases.
+
+All expressions are enabled by default, but most can be disabled by setting
+`spark.comet.expression.EXPRNAME.enabled=false`, where `EXPRNAME` is the expression name as specified in
+the following tables, such as `Length`, or `StartsWith`. See the [Comet Configuration Guide] for a full list
+of expressions that be disabled.
+
+## Conditional Expressions
+
+| Expression | SQL |
+| ---------- | ------------------------------------------- |
+| CaseWhen | `CASE WHEN expr THEN expr ELSE expr END` |
+| If | `IF(predicate_expr, true_expr, false_expr)` |
+
+## Predicate Expressions
+
+| Expression | SQL |
+| ------------------ | ------------- |
+| And | `AND` |
+| EqualTo | `=` |
+| EqualNullSafe | `<=>` |
+| GreaterThan | `>` |
+| GreaterThanOrEqual | `>=` |
+| LessThan | `<` |
+| LessThanOrEqual | `<=` |
+| In | `IN` |
+| IsNotNull | `IS NOT NULL` |
+| IsNull | `IS NULL` |
+| InSet | `IN (...)` |
+| Not | `NOT` |
+| Or | `OR` |
+
+## String Functions
+
+| Expression |
+| --------------- |
+| Ascii |
+| BitLength |
+| Chr |
+| Concat |
+| ConcatWs |
+| Contains |
+| EndsWith |
+| InitCap |
+| Left |
+| Length |
+| Like |
+| Lower |
+| OctetLength |
+| Reverse |
+| RLike |
+| StartsWith |
+| StringInstr |
+| StringRepeat |
+| StringReplace |
+| StringLPad |
+| StringRPad |
+| StringSpace |
+| StringTranslate |
+| StringTrim |
+| StringTrimBoth |
+| StringTrimLeft |
+| StringTrimRight |
+| Substring |
+| Upper |
+
+## JSON Functions
+
+| Expression |
+| ------------- |
+| GetJsonObject |
+
+## Date/Time Functions
+
+| Expression | SQL |
+| -------------- | ---------------------------- |
+| DateAdd | `date_add` |
+| DateDiff | `datediff` |
+| DateFormat | `date_format` |
+| DateSub | `date_sub` |
+| DatePart | `date_part(field, source)` |
+| Days | `days` |
+| Extract | `extract(field FROM source)` |
+| FromUnixTime | `from_unixtime` |
+| Hour | `hour` |
+| LastDay | `last_day` |
+| Minute | `minute` |
+| Second | `second` |
+| TruncDate | `trunc` |
+| TruncTimestamp | `date_trunc` |
+| UnixDate | `unix_date` |
+| UnixTimestamp | `unix_timestamp` |
+| Year | `year` |
+| Month | `month` |
+| DayOfMonth | `day`/`dayofmonth` |
+| DayOfWeek | `dayofweek` |
+| WeekDay | `weekday` |
+| DayOfYear | `dayofyear` |
+| WeekOfYear | `weekofyear` |
+| Quarter | `quarter` |
+
+## Math Expressions
+
+| Expression | SQL |
+| -------------- | --------- |
+| Abs | `abs` |
+| Acos | `acos` |
+| Add | `+` |
+| Asin | `asin` |
+| Atan | `atan` |
+| Atan2 | `atan2` |
+| BRound | `bround` |
+| Ceil | `ceil` |
+| Cos | `cos` |
+| Cosh | `cosh` |
+| Cot | `cot` |
+| Divide | `/` |
+| Exp | `exp` |
+| Expm1 | `expm1` |
+| Floor | `floor` |
+| Hex | `hex` |
+| IntegralDivide | `div` |
+| IsNaN | `isnan` |
+| Log | `log` |
+| Log2 | `log2` |
+| Log10 | `log10` |
+| Multiply | `*` |
+| Pow | `power` |
+| Rand | `rand` |
+| Randn | `randn` |
+| Remainder | `%` |
+| Round | `round` |
+| Signum | `signum` |
+| Sin | `sin` |
+| Sinh | `sinh` |
+| Sqrt | `sqrt` |
+| Subtract | `-` |
+| Tan | `tan` |
+| Tanh | `tanh` |
+| TryAdd | `try_add` |
+| TryDivide | `try_div` |
+| TryMultiply | `try_mul` |
+| TrySubtract | `try_sub` |
+| UnaryMinus | `-` |
+| Unhex | `unhex` |
+
+## Hashing Functions
+
+| Expression |
+| ----------- |
+| Md5 |
+| Murmur3Hash |
+| Sha1 |
+| Sha2 |
+| XxHash64 |
+
+## Bitwise Expressions
+
+| Expression | SQL |
+| ------------ | ---- |
+| BitwiseAnd | `&` |
+| BitwiseCount | |
+| BitwiseGet | |
+| BitwiseOr | `\|` |
+| BitwiseNot | `~` |
+| BitwiseXor | `^` |
+| ShiftLeft | `<<` |
+| ShiftRight | `>>` |
+
+## Aggregate Expressions
+
+| Expression | SQL |
+| ------------- | ---------- |
+| Average | |
+| BitAndAgg | |
+| BitOrAgg | |
+| BitXorAgg | |
+| BoolAnd | `bool_and` |
+| BoolOr | `bool_or` |
+| CollectSet | |
+| Corr | |
+| Count | |
+| CovPopulation | |
+| CovSample | |
+| First | |
+| Last | |
+| Max | |
+| Min | |
+| StddevPop | |
+| StddevSamp | |
+| Sum | |
+| VariancePop | |
+| VarianceSamp | |
+
+## Window Functions
+
+```{warning}
+Window support is disabled by default due to known correctness issues. Tracking issue: [#2721](https://github.com/apache/datafusion-comet/issues/2721).
+```
+
+Comet supports using the following aggregate functions within window contexts with PARTITION BY and ORDER BY clauses.
+
+| Expression |
+| ---------- |
+| Count |
+| Max |
+| Min |
+| Sum |
+
+**Note:** Dedicated window functions such as `rank`, `dense_rank`, `row_number`, `lag`, `lead`, `ntile`, `cume_dist`, `percent_rank`, and `nth_value` are not currently supported and will fall back to Spark.
+
+## Array Expressions
+
+| Expression |
+| -------------- |
+| ArrayAppend |
+| ArrayCompact |
+| ArrayContains |
+| ArrayDistinct |
+| ArrayExcept |
+| ArrayFilter |
+| ArrayInsert |
+| ArrayIntersect |
+| ArrayJoin |
+| ArrayMax |
+| ArrayMin |
+| ArrayRemove |
+| ArrayRepeat |
+| ArrayUnion |
+| ArraysOverlap |
+| CreateArray |
+| ElementAt |
+| Flatten |
+| GetArrayItem |
+
+## Map Expressions
+
+| Expression |
+| ------------- |
+| GetMapValue |
+| MapKeys |
+| MapEntries |
+| MapValues |
+| MapFromArrays |
+| StringToMap |
+
+## Struct Expressions
+
+| Expression |
+| -------------------- |
+| CreateNamedStruct |
+| GetArrayStructFields |
+| GetStructField |
+| JsonToStructs |
+| StructsToJson |
+
+## Conversion Expressions
+
+| Expression |
+| ---------- |
+| Cast |
+
+## SortOrder
+
+| Expression |
+| ---------- |
+| NullsFirst |
+| NullsLast |
+| Ascending |
+| Descending |
+
+## Other
+
+| Expression |
+| ---------------------------- |
+| Alias |
+| AttributeReference |
+| BloomFilterMightContain |
+| Coalesce |
+| CheckOverflow |
+| KnownFloatingPointNormalized |
+| Literal |
+| MakeDecimal |
+| MonotonicallyIncreasingID |
+| NormalizeNaNAndZero |
+| PromotePrecision |
+| RegExpReplace |
+| ScalarSubquery |
+| SparkPartitionID |
+| ToPrettyString |
+| UnscaledValue |
+
+[Comet Configuration Guide]: configs.md
+[Comet Compatibility Guide]: compatibility/expressions/index.md
diff --git a/_sources/user-guide/0.15/iceberg.md.txt b/_sources/user-guide/0.16/iceberg.md.txt
similarity index 82%
copy from _sources/user-guide/0.15/iceberg.md.txt
copy to _sources/user-guide/0.16/iceberg.md.txt
index 4dbd085c2..c9cf80304 100644
--- a/_sources/user-guide/0.15/iceberg.md.txt
+++ b/_sources/user-guide/0.16/iceberg.md.txt
@@ -25,13 +25,17 @@ Comet's native Iceberg reader relies on reflection to extract `FileScanTask`s fr
then serialized to Comet's native execution engine (see
[PR #2528](https://github.com/apache/datafusion-comet/pull/2528)).
-The example below uses Spark's package downloader to retrieve Comet 0.14.0 and Iceberg
+The example below uses Spark's package downloader to retrieve Comet 0.16.0 and Iceberg
1.8.1, but Comet has been tested with Iceberg 1.5, 1.7, 1.8, 1.9, and 1.10. The native Iceberg
reader is enabled by default. To disable it, set `spark.comet.scan.icebergNative.enabled=false`.
+The example uses the Spark 3.5 / Scala 2.12 build of Comet; substitute the Comet artifact
+matching your Spark and Scala versions (Comet also ships Spark 3.5 / Scala 2.13 and Spark
+4.0/4.1 / Scala 2.13 jars; see the [installation guide](installation.md) for the full list).
+
```shell
$SPARK_HOME/bin/spark-shell \
- --packages org.apache.datafusion:comet-spark-spark3.5_2.12:0.14.0,org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.iceberg:iceberg-core:1.8.1 \
+ --packages org.apache.datafusion:comet-spark-spark3.5_2.12:0.16.0,org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.iceberg:iceberg-core:1.8.1 \
--repositories https://repo1.maven.org/maven2/ \
--conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
--conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkCatalog \
@@ -39,7 +43,6 @@ $SPARK_HOME/bin/spark-shell \
--conf spark.sql.catalog.spark_catalog.warehouse=/tmp/warehouse \
--conf spark.plugins=org.apache.spark.CometPlugin \
--conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
- --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
--conf spark.comet.explainFallback.enabled=true \
--conf spark.memory.offHeap.enabled=true \
--conf spark.memory.offHeap.size=2g
@@ -106,7 +109,7 @@ configure Spark to use a REST catalog with Comet's native Iceberg scan:
```shell
$SPARK_HOME/bin/spark-shell \
- --packages org.apache.datafusion:comet-spark-spark3.5_2.12:0.14.0,org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.iceberg:iceberg-core:1.8.1 \
+ --packages org.apache.datafusion:comet-spark-spark3.5_2.12:0.16.0,org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.iceberg:iceberg-core:1.8.1 \
--repositories https://repo1.maven.org/maven2/ \
--conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
--conf spark.sql.catalog.rest_cat=org.apache.iceberg.spark.SparkCatalog \
@@ -115,7 +118,6 @@ $SPARK_HOME/bin/spark-shell \
--conf spark.sql.catalog.rest_cat.warehouse=/tmp/warehouse \
--conf spark.plugins=org.apache.spark.CometPlugin \
--conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
- --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
--conf spark.comet.explainFallback.enabled=true \
--conf spark.memory.offHeap.enabled=true \
--conf spark.memory.offHeap.size=2g
@@ -141,3 +143,11 @@ The following scenarios will fall back to Spark's native Iceberg reader:
- Scans with residual filters using `truncate`, `bucket`, `year`, `month`, `day`, or `hour`
transform functions (partition pruning still works, but row-level filtering of these
transforms falls back)
+- Dynamic Partition Pruning under Adaptive Query Execution (non-AQE DPP is supported);
+ see [#3510](https://github.com/apache/datafusion-comet/issues/3510)
+
+### Task input metrics
+
+The native Iceberg reader populates Spark's task-level `inputMetrics.bytesRead` (visible in the Spark UI Stages tab) using the `bytes_read` counter from iceberg-rust's `ScanMetrics`. This counter includes bytes read from both data files and delete files.
+
+Iceberg Java does not explicitly report `bytesRead` to Spark's task input metrics. On the iceberg Java path, any `bytesRead` value comes from Hadoop's filesystem-level I/O counters, not from Iceberg itself. Because Comet's native reader and the Hadoop filesystem use different counting mechanisms, the exact byte counts will differ between the two paths.
diff --git a/_sources/user-guide/0.12/index.rst.txt b/_sources/user-guide/0.16/index.rst.txt
similarity index 64%
rename from _sources/user-guide/0.12/index.rst.txt
rename to _sources/user-guide/0.16/index.rst.txt
index 1ec3f57d6..f683fa069 100644
--- a/_sources/user-guide/0.12/index.rst.txt
+++ b/_sources/user-guide/0.16/index.rst.txt
@@ -19,13 +19,23 @@
:alt: DataFusion Comet Logo
================================
-Comet 0.12.0 User Guide
+Comet 0.16.0 User Guide
================================
-.. _toc.user-guide-links-0.12.0:
+This guide covers Comet 0.16.0: how to install it, build it from source, configure it for
+your Spark deployment, and get the best results from it. It also documents the data sources, data
+types, operators, and expressions that Comet supports, along with a compatibility guide describing
+known differences from Apache Spark.
+
+Operational topics include reading and understanding Comet query plans, tuning, available metrics,
+and integration guides for Apache Iceberg and Kubernetes. Select a topic from the navigation menu
+to read more.
+
+.. _toc.user-guide-links-0.16.0:
.. toctree::
:maxdepth: 1
- :caption: Comet 0.12.0 User Guide
+ :caption: Comet 0.16.0 User Guide
+ :hidden:
Installing Comet <installation>
Building From Source <source>
@@ -34,7 +44,8 @@ Comet 0.12.0 User Guide
Supported Operators <operators>
Supported Expressions <expressions>
Configuration Settings <configs>
- Compatibility Guide <compatibility>
+ Compatibility Guide <compatibility/index>
+ Understanding Comet Plans <understanding-comet-plans>
Tuning Guide <tuning>
Metrics Guide <metrics>
Iceberg Guide <iceberg>
diff --git a/_sources/user-guide/0.12/installation.md.txt b/_sources/user-guide/0.16/installation.md.txt
similarity index 52%
rename from _sources/user-guide/0.12/installation.md.txt
rename to _sources/user-guide/0.16/installation.md.txt
index 4096faeec..81e608f86 100644
--- a/_sources/user-guide/0.12/installation.md.txt
+++ b/_sources/user-guide/0.16/installation.md.txt
@@ -16,9 +16,6 @@
specific language governing permissions and limitations
under the License.
-->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
# Installing DataFusion Comet
@@ -28,12 +25,22 @@ Make sure the following requirements are met and software installed on your mach
### Supported Operating Systems
-- Linux
-- Apple macOS (Intel and Apple Silicon)
+The published Comet jar files in Maven Central bundle native libraries for Linux only (amd64 and arm64). macOS
+users must [build from source](source.md).
+
+| Operating System | Published Maven Jars | Build from Source |
+| --------------------------- | -------------------- | ----------------- |
+| Linux (amd64) | Yes | Yes |
+| Linux (arm64) | Yes | Yes |
+| Apple macOS (Apple Silicon) | No | Yes |
### Supported Spark Versions
-Comet 0.12.0 supports the following versions of Apache Spark.
+Comet 0.16.0 supports the following versions of Apache Spark. Refer to the [Spark Version Compatibility] page
+in the [Compatibility Guide] for more information, such as known limitations per Spark version.
+
+[Spark Version Compatibility]: compatibility/spark-versions.md
+[Compatibility Guide]: compatibility/index.md
We recommend only using Comet with Spark versions where we currently have both Comet and Spark tests enabled in CI.
Other versions may work well enough for development and evaluation purposes.
@@ -41,43 +48,50 @@ Other versions may work well enough for development and evaluation purposes.
| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
| ------------- | ------------ | ------------- | ----------------- | --------------------- |
| 3.4.3 | 11/17 | 2.12/2.13 | Yes | Yes |
-| 3.5.5 | 11/17 | 2.12/2.13 | Yes | No |
-| 3.5.6 | 11/17 | 2.12/2.13 | Yes | No |
-| 3.5.7 | 11/17 | 2.12/2.13 | Yes | Yes |
+| 3.5.8 | 11/17 | 2.12/2.13 | Yes | Yes |
+| 4.0.2 | 17/21 | 2.13 | Yes | Yes |
+| 4.1.1 | 17/21 | 2.13 | Yes | Yes |
Note that we do not test the full matrix of supported Java and Scala versions in CI for every Spark version.
Experimental support is provided for the following versions of Apache Spark and is intended for development/testing
use only and should not be used in production yet.
-| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
-| ------------- | ------------ | ------------- | ----------------- | --------------------- |
-| 4.0.1 | 17 | 2.13 | Yes | Yes |
+| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
+| -------------- | ------------ | ------------- | ----------------- | --------------------- |
+| 4.2.0-preview4 | 17 | 2.13 | No | No |
Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by
Cloud Service Providers.
## Using a Published JAR File
+
+
Comet jar files are available in [Maven Central](https://central.sonatype.com/namespace/org.apache.datafusion) for amd64 and arm64 architectures for Linux. For Apple macOS, it
is currently necessary to build from source.
-Here are the direct links for downloading the Comet 0.12.0 jar file. Note that these links are not valid if
-you are viewing the documentation for the latest `SNAPSHOT` release, or for the latest release while it is still going
-through the release vote. Release candidate jars can be found [here](https://repository.apache.org/#nexus-search;quick~org.apache.datafusion).
+For performance reasons, published Comet jar files target baseline CPUs available in modern data centers. For example,
+the amd64 build uses the `x86-64-v3` target that adds CPU instructions (_e.g._, AVX2) common after 2013. Similarly, the
+arm64 build uses the `neoverse-n1` target, which is a common baseline for ARM cores found in AWS (Graviton2+), GCP, and
+Azure after 2019. If the Comet library fails for SIGILL (illegal instruction), please open an issue on the GitHub
+repository describing your environment, and [build from source] for your target architecture.
-- [Comet plugin for Spark 3.4 / Scala 2.12](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.4_2.12/0.12.0/comet-spark-spark3.4_2.12-0.12.0.jar)
-- [Comet plugin for Spark 3.4 / Scala 2.13](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.4_2.13/0.12.0/comet-spark-spark3.4_2.13-0.12.0.jar)
-- [Comet plugin for Spark 3.5 / Scala 2.12](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.5_2.12/0.12.0/comet-spark-spark3.5_2.12-0.12.0.jar)
-- [Comet plugin for Spark 3.5 / Scala 2.13](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.5_2.13/0.12.0/comet-spark-spark3.5_2.13-0.12.0.jar)
-- [Comet plugin for Spark 4.0 / Scala 2.13 (Experimental)](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark4.0_2.13/0.12.0/comet-spark-spark4.0_2.13-0.12.0.jar)
+Here are the direct links for downloading the Comet 0.16.0 jar file.
+
+- [Comet plugin for Spark 3.4 / Scala 2.12](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.4_2.12/0.16.0/comet-spark-spark3.4_2.12-0.16.0.jar)
+- [Comet plugin for Spark 3.4 / Scala 2.13](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.4_2.13/0.16.0/comet-spark-spark3.4_2.13-0.16.0.jar)
+- [Comet plugin for Spark 3.5 / Scala 2.12](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.5_2.12/0.16.0/comet-spark-spark3.5_2.12-0.16.0.jar)
+- [Comet plugin for Spark 3.5 / Scala 2.13](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.5_2.13/0.16.0/comet-spark-spark3.5_2.13-0.16.0.jar)
+- [Comet plugin for Spark 4.0 / Scala 2.13](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark4.0_2.13/0.16.0/comet-spark-spark4.0_2.13-0.16.0.jar)
+- [Comet plugin for Spark 4.1 / Scala 2.13](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark4.1_2.13/0.16.0/comet-spark-spark4.1_2.13-0.16.0.jar)
## Building from source
-Refer to the [Building from Source] guide for instructions from building Comet from source, either from official
+Refer to the [Building from source] guide for instructions from building Comet from source, either from official
source releases, or from the latest code in the GitHub repository.
-[Building from Source]: source.md
+[Building from source]: source.md
## Deploying to Kubernetes
@@ -88,7 +102,7 @@ See the [Comet Kubernetes Guide](kubernetes.md) guide.
Make sure `SPARK_HOME` points to the same Spark version as Comet was built for.
```shell
-export COMET_JAR=spark/target/comet-spark-spark3.5_2.12-0.12.0.jar
+export COMET_JAR=spark/target/comet-spark-spark4.1_2.13-0.16.0.jar
$SPARK_HOME/bin/spark-shell \
--jars $COMET_JAR \
@@ -98,7 +112,7 @@ $SPARK_HOME/bin/spark-shell \
--conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
--conf spark.comet.explainFallback.enabled=true \
--conf spark.memory.offHeap.enabled=true \
- --conf spark.memory.offHeap.size=16g
+ --conf spark.memory.offHeap.size=4g
```
### Verify Comet enabled for Spark SQL query
@@ -109,6 +123,16 @@ Create a test Parquet source
scala> (0 until 10).toDF("a").write.mode("overwrite").parquet("/tmp/test")
```
+Comet will log output similar to:
+
+```shell
+INFO core/src/lib.rs: Comet native library version 0.16.0 initialized
+WARN CometExecRule: Comet cannot execute some parts of this plan natively (set spark.comet.explainFallback.enabled=false to disable this logging):
+ Execute InsertIntoHadoopFsRelationCommand [COMET: Native support for operator DataWritingCommandExec is disabled. Set spark.comet.parquet.write.enabled=true to enable it.]
++- WriteFiles
+ +- LocalTableScan [COMET: Native support for operator LocalTableScanExec is disabled. Set spark.comet.exec.localTableScan.enabled=true to enable it.]
+```
+
Query the data from the test source and check:
- INFO message shows the native Comet library has been initialized.
@@ -117,24 +141,15 @@ Query the data from the test source and check:
```scala
scala> spark.read.parquet("/tmp/test").createOrReplaceTempView("t1")
scala> spark.sql("select * from t1 where a > 5").explain
-INFO src/lib.rs: Comet native library initialized
-== Physical Plan ==
- *(1) ColumnarToRow
- +- CometFilter [a#14], (isnotnull(a#14) AND (a#14 > 5))
- +- CometScan parquet [a#14] Batched: true, DataFilters: [isnotnull(a#14), (a#14 > 5)],
- Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/tmp/test], PartitionFilters: [],
- PushedFilters: [IsNotNull(a), GreaterThan(a,5)], ReadSchema: struct<a:int>
```
-With the configuration `spark.comet.explainFallback.enabled=true`, Comet will log any reasons that prevent a plan from
-being executed natively.
+Comet will log output similar to:
-```scala
-scala> Seq(1,2,3,4).toDF("a").write.parquet("/tmp/test.parquet")
-WARN CometSparkSessionExtensions$CometExecRule: Comet cannot execute some parts of this plan natively because:
- - LocalTableScan is not supported
- - WriteFiles is not supported
- - Execute InsertIntoHadoopFsRelationCommand is not supported
+```shell
+== Physical Plan ==
+CometNativeColumnarToRow
++- CometFilter [a#6], (isnotnull(a#6) AND (a#6 > 5))
+ +- CometNativeScan parquet [a#6] Batched: true, DataFilters: [isnotnull(a#6), (a#6 > 5)], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/tmp/test], PartitionFilters: [], PushedFilters: [IsNotNull(a), GreaterThan(a,5)], ReadSchema: struct<a:int>
```
## Additional Configuration
@@ -143,8 +158,8 @@ Depending on your deployment mode you may also need to set the driver & executor
explicitly contain Comet otherwise Spark may use a different class-loader for the Comet components than its internal
components which will then fail at runtime. For example:
-```
---driver-class-path spark/target/comet-spark-spark3.5_2.12-0.12.0.jar
+```shell
+--driver-class-path spark/target/comet-spark-spark4.1_2.13-0.16.0.jar
```
Some cluster managers may require additional configuration, see <https://spark.apache.org/docs/latest/cluster-overview.html>
diff --git a/_sources/user-guide/0.12/kubernetes.md.txt b/_sources/user-guide/0.16/kubernetes.md.txt
similarity index 93%
rename from _sources/user-guide/0.12/kubernetes.md.txt
rename to _sources/user-guide/0.16/kubernetes.md.txt
index c86d983f5..94735dfef 100644
--- a/_sources/user-guide/0.12/kubernetes.md.txt
+++ b/_sources/user-guide/0.16/kubernetes.md.txt
@@ -16,9 +16,6 @@
specific language governing permissions and limitations
under the License.
-->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
# Comet Kubernetes Support
@@ -72,30 +69,30 @@ metadata:
spec:
type: Scala
mode: cluster
- image: apache/datafusion-comet:0.7.0-spark3.5.5-scala2.12-java11
+ image: apache/datafusion-comet:0.16.0-spark3.5.5-scala2.12-java11
imagePullPolicy: IfNotPresent
mainClass: org.apache.spark.examples.SparkPi
- mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.5.jar
+ mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.13-4.1.1.jar
sparkConf:
- "spark.executor.extraClassPath": "/opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar"
- "spark.driver.extraClassPath": "/opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar"
+ "spark.executor.extraClassPath": "/opt/spark/jars/comet-spark-spark3.5_2.12-0.16.0.jar"
+ "spark.driver.extraClassPath": "/opt/spark/jars/comet-spark-spark3.5_2.12-0.16.0.jar"
"spark.plugins": "org.apache.spark.CometPlugin"
"spark.comet.enabled": "true"
"spark.comet.exec.enabled": "true"
"spark.comet.exec.shuffle.enabled": "true"
"spark.comet.exec.shuffle.mode": "auto"
"spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
- sparkVersion: 3.5.6
+ sparkVersion: 4.1.1
driver:
labels:
- version: 3.5.6
+ version: 4.1.1
cores: 1
coreLimit: 1200m
memory: 512m
serviceAccount: spark-operator-spark
executor:
labels:
- version: 3.5.6
+ version: 4.1.1
instances: 1
cores: 1
coreLimit: 1200m
diff --git a/_sources/user-guide/0.12/metrics.md.txt b/_sources/user-guide/0.16/metrics.md.txt
similarity index 72%
rename from _sources/user-guide/0.12/metrics.md.txt
rename to _sources/user-guide/0.16/metrics.md.txt
index 83169dfa2..0579b3081 100644
--- a/_sources/user-guide/0.12/metrics.md.txt
+++ b/_sources/user-guide/0.16/metrics.md.txt
@@ -16,9 +16,6 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
# Comet Metrics
@@ -67,3 +64,26 @@ Here is a guide to some of the native metrics.
| `ipc_time` | Time to encode batches in IPC format and compress using ZSTD. |
| `mempool_time` | Time interacting with memory pool. |
| `write_time` | Time spent writing bytes to disk. |
+
+## Task-Level Input Metrics on Spark 4.1+
+
+Comet's native scans set `inputMetrics.bytesRead` to the actual file IO performed by the
+DataFusion parquet reader (`bytes_scanned`). This is the truthful number you would see at the
+filesystem layer.
+
+Spark 4.1 changed its own parquet reader to pre-open the `SeekableInputStream` and read the file
+footer outside the `FileScanRDD.compute()` thread. Spark's `inputMetrics.bytesRead` is updated
+from a Hadoop FileSystem thread-local byte counter that only captures reads on the
+`compute()` thread, so reads serviced by the pre-opened stream's internal buffer go uncounted.
+The under-count is largest when the file fits in the pre-fetched buffer (tiny files, unit test
+sizes) and shrinks as files grow large enough that subsequent row-group reads cross the buffer
+and trigger fresh FS reads on the `compute()` thread.
+
+This is purely an observability difference: `inputMetrics.bytesRead` is reported to listeners
+and the Spark UI but is not consumed by the planner, the optimizer, or AQE, so the discrepancy
+does not affect query plans, partitioning, or correctness. Records read (`recordsRead`) is
+unaffected and remains exactly equal between Comet and Spark.
+
+If you compare Comet's `bytesRead` against vanilla Spark's on Spark 4.1+ (via the Spark UI or
+the REST API), expect Comet's number to be substantially larger for small files, and closer to
+Spark's for large files. Comet's value reflects what the storage layer actually delivered.
diff --git a/_sources/user-guide/0.15/operators.md.txt b/_sources/user-guide/0.16/operators.md.txt
similarity index 98%
copy from _sources/user-guide/0.15/operators.md.txt
copy to _sources/user-guide/0.16/operators.md.txt
index 77ba84e4f..1b8f78d9c 100644
--- a/_sources/user-guide/0.15/operators.md.txt
+++ b/_sources/user-guide/0.16/operators.md.txt
@@ -45,4 +45,4 @@ not supported by Comet will fall back to regular Spark execution.
| UnionExec | Yes | |
| WindowExec | No | Disabled by default due to known correctness issues. |
-[Comet Compatibility Guide]: compatibility.md
+[Comet Compatibility Guide]: compatibility/operators.md
diff --git a/_sources/user-guide/0.12/source.md.txt b/_sources/user-guide/0.16/source.md.txt
similarity index 75%
rename from _sources/user-guide/0.12/source.md.txt
rename to _sources/user-guide/0.16/source.md.txt
index 5b3e83b14..c7d45c7cc 100644
--- a/_sources/user-guide/0.12/source.md.txt
+++ b/_sources/user-guide/0.16/source.md.txt
@@ -16,9 +16,6 @@
specific language governing permissions and limitations
under the License.
-->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
# Building Comet From Source
@@ -26,24 +23,27 @@ It is sometimes preferable to build from source for a specific platform.
## Using a Published Source Release
+
+
Official source releases can be downloaded from https://dist.apache.org/repos/dist/release/datafusion/
```console
# Pick the latest version
-export COMET_VERSION=0.12.0
+export COMET_VERSION=0.16.0
# Download the tarball
-curl -O "https://dist.apache.org/repos/dist/release/datafusion/datafusion-comet-0.12.0/apache-datafusion-comet-0.12.0.tar.gz"
+curl -O "https://dist.apache.org/repos/dist/release/datafusion/datafusion-comet-0.16.0/apache-datafusion-comet-0.16.0.tar.gz"
# Unpack
-tar -xzf apache-datafusion-comet-0.12.0.tar.gz
-cd apache-datafusion-comet-0.12.0
+tar -xzf apache-datafusion-comet-0.16.0.tar.gz
+cd apache-datafusion-comet-0.16.0
```
Build
```console
-make release-nogit PROFILES="-Pspark-3.5"
+make release-nogit PROFILES="-Pspark-4.1"
```
+
## Building from the GitHub repository
Clone the repository:
@@ -56,17 +56,17 @@ Build Comet for a specific Spark version:
```console
cd datafusion-comet
-make release PROFILES="-Pspark-3.5"
+make release PROFILES="-Pspark-4.1"
```
-Note that the project builds for Scala 2.12 by default but can be built for Scala 2.13 using an additional profile:
+Note that the project builds for Scala 2.13 by default but can be built for Scala 2.12 using an additional profile:
```console
-make release PROFILES="-Pspark-3.5 -Pscala-2.13"
+make release PROFILES="-Pspark-3.5 -Pscala-2.12"
```
To build Comet from the source distribution on an isolated environment without an access to `github.com` it is necessary to disable `git-commit-id-maven-plugin`, otherwise you will face errors that there is no access to the git during the build process. In that case you may use:
```console
-make release-nogit PROFILES="-Pspark-3.5"
+make release-nogit PROFILES="-Pspark-4.1"
```
diff --git a/_sources/user-guide/0.12/tuning.md.txt b/_sources/user-guide/0.16/tuning.md.txt
similarity index 76%
rename from _sources/user-guide/0.12/tuning.md.txt
rename to _sources/user-guide/0.16/tuning.md.txt
index 3c4a66c21..a394b4a4c 100644
--- a/_sources/user-guide/0.12/tuning.md.txt
+++ b/_sources/user-guide/0.16/tuning.md.txt
@@ -16,9 +16,6 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-```{warning}
-This is **out-of-date** documentation. The latest Comet release is version 0.15.0.
-```
# Comet Tuning Guide
@@ -64,7 +61,12 @@ The valid pool types are:
- `fair_unified` (default when `spark.memory.offHeap.enabled=true` is set)
- `greedy_unified`
-The `fair_unified` pool types prevents operators from using more than an even fraction of the available memory
+Both pool types are shared across all native execution contexts within the same Spark task. When
+Comet executes a shuffle, it runs two native execution contexts concurrently (e.g. one for
+pre-shuffle operators and one for the shuffle writer). The shared pool ensures that the combined
+memory usage stays within the per-task limit.
+
+The `fair_unified` pool prevents operators from using more than an even fraction of the available memory
(i.e. `pool_size / num_reservations`). This pool works best when you know beforehand
the query has multiple operators that will likely all need to spill. Sometimes it will cause spills even
when there is sufficient memory in order to leave enough memory for other operators.
@@ -152,6 +154,32 @@ partitioning keys. Columns that are not partitioning keys may contain complex ty
Comet Columnar shuffle is JVM-based and supports `HashPartitioning`, `RoundRobinPartitioning`, `RangePartitioning`, and
`SinglePartitioning`. This shuffle implementation supports complex data types as partitioning keys.
+By default, Comet will convert a Spark `ShuffleExchangeExec` to columnar shuffle even when the shuffle's child is a
+non-Comet (Spark) plan. The benefit is that the next query stage can start as native Comet execution, since the
+shuffle output is already in Arrow format. The cost is a row to columnar conversion at the shuffle boundary on the
+write side. To restrict columnar shuffle to cases where the child is already a Comet plan, set
+`spark.comet.exec.shuffle.convertFromSparkPlan.enabled=false`. Shuffles whose child is a Spark plan will then be left
+as native Spark shuffles, which avoids the row to columnar conversion but means the downstream stage will also start
+on Spark.
+
+#### Automatic Revert to Spark Shuffle
+
+When a Comet columnar shuffle ends up between two non-Comet operators (for example, a partial/final hash aggregate
+pair that Comet could not convert), Comet reverts it to Spark's built-in shuffle. Keeping columnar shuffle between
+two row-based operators would add `row -> Arrow -> shuffle -> Arrow -> row` conversions with no Comet consumer on
+either side to benefit from columnar output.
+
+This shifts the affected shuffles from Comet's off-heap memory pool back to the JVM execution memory pool. Clusters
+tuned for a small JVM heap may see `ExternalSorter` spills on queries where this revert fires. Shuffle I/O may also
+grow marginally because Spark's row-based serializer generally compresses less well than Comet's Arrow IPC format.
+
+Each revert is logged at `INFO` level on the driver as `Reverting Comet columnar shuffle to Spark shuffle between
+<parent> and <child>`, which lets you correlate any unexpected behavior with this optimization.
+
+This optimization is enabled by default and can be disabled by setting
+`spark.comet.exec.shuffle.revertRedundantColumnar.enabled=false`, in which case Comet will keep the columnar shuffle
+even when both its parent and child are non-Comet operators.
+
### Shuffle Compression
By default, Spark compresses shuffle files using LZ4 compression. Comet overrides this behavior with ZSTD compression.
@@ -160,14 +188,5 @@ certain environments, such as single-node setups with fast NVMe drives, at the e
## Explain Plan
-### Extended Explain
-
-With Spark 4.0.0 and newer, Comet can provide extended explain plan information in the Spark UI. Currently this lists
-reasons why Comet may not have been enabled for specific operations.
-To enable this, in the Spark configuration, set the following:
-
-```shell
--c spark.sql.extendedExplainProviders=org.apache.comet.ExtendedExplainInfo
-```
-
-This will add a section to the detailed plan displayed in the Spark SQL UI page.
+For an explanation of Comet plan output, the configs that control it, and how
+fallback to Spark works, see [Understanding Comet Plans](understanding-comet-plans.md).
diff --git a/_sources/user-guide/0.16/understanding-comet-plans.md.txt b/_sources/user-guide/0.16/understanding-comet-plans.md.txt
new file mode 100644
index 000000000..f66f38520
--- /dev/null
+++ b/_sources/user-guide/0.16/understanding-comet-plans.md.txt
@@ -0,0 +1,232 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Understanding Comet Plans
+
+This guide explains how to read a Spark query plan once Comet is enabled, what
+happens when parts of a plan fall back to Spark, and which configs to use to
+inspect that behavior.
+
+## Overview
+
+When Comet is enabled, the `CometSparkSessionExtensions` rules walk the
+physical plan bottom-up and replace Spark operators with Comet equivalents
+where possible. Consecutive native operators are combined into a single block
+that is serialized as protobuf and executed by DataFusion on the executor.
+Operators that Comet does not support remain as their original Spark form.
+
+As a result, a plan can mix three kinds of nodes:
+
+- **`Comet*` nodes** that run natively in Rust (for example `CometProject`,
+ `CometHashAggregate`).
+- **`Comet*` nodes that run on the JVM** but are still part of the Comet
+ pipeline (for example `CometBroadcastExchange`, `CometColumnarExchange`).
+- **Standard Spark nodes** (for example `Project`, `HashAggregate`) where
+ Comet either does not support the operator or has fallen back due to an
+ unsupported expression, data type, or configuration.
+
+Wherever data crosses between columnar and row-based execution, Comet inserts
+a transition node such as `CometColumnarToRow` or `CometSparkRowToColumnar`.
+
+## Reading a Plan
+
+You can print a plan with `df.explain("formatted")` or `EXPLAIN FORMATTED <sql>`, and
+the same plan is shown in the Spark SQL UI. When reading a plan, look for:
+
+- **Node prefix.** `Comet*` nodes are accelerated by Comet. Anything without
+ the prefix is unmodified Spark.
+- **Transitions.** `CometColumnarToRow`, `CometNativeColumnarToRow`, and
+ `CometSparkRowToColumnar` mark boundaries between columnar Comet execution
+ and row-based Spark execution. Frequent transitions usually indicate
+ fallback inside the plan.
+- **Exchange type.** `CometExchange` is the native shuffle path,
+ `CometColumnarExchange` is the JVM columnar shuffle path, and a plain
+ `Exchange` means Spark shuffle. See [Shuffle Operators](#shuffle-operators)
+ below.
+
+## Fallback
+
+A "fallback" happens when Comet cannot translate part of a plan into native
+execution. Fallback can be partial (a subtree falls back while the rest stays
+native) or full (no Comet nodes appear).
+
+Common reasons:
+
+- The Spark operator is not supported by Comet.
+- An expression inside an otherwise supported operator is not supported, or
+ is marked incompatible and the per-expression opt-in
+ `spark.comet.expression.<ExpressionName>.allowIncompatible=true` is not
+ set. Operators have an equivalent
+ `spark.comet.operator.<OperatorName>.allowIncompatible` opt-in.
+- A data type is not supported by the operator.
+- A configuration setting disables a specific operator or expression.
+
+See [Supported Spark Operators](operators.md) and [Supported Expressions](expressions.md)
+for current coverage, and the [Compatibility Guide](compatibility/index.md) for
+incompatibility details.
+
+## Configs for Inspecting Plans and Fallback
+
+Comet provides four configs for understanding what is happening in a plan.
+They serve different purposes and produce output in different places.
+
+| Config | Output destination | What you see |
+| ---------------------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------- |
+| `spark.comet.explainFallback.enabled` | Driver log (only when fallback) | A WARN with the list of reasons each query stage could not run natively. |
+| `spark.comet.logFallbackReasons.enabled` | Driver log | One WARN per fallback reason as it is encountered, without surrounding plan context. |
+| `spark.comet.explain.format` | Spark SQL UI (Spark 4.0 and newer) | Annotated plan or fallback-reason list, depending on `verbose` (default) or `fallback` value. |
+| `spark.comet.explain.native.enabled` | Executor logs, per task | The DataFusion native plan with metrics, useful for inspecting native execution. |
+
+### `spark.comet.explainFallback.enabled`
+
+Logs a single WARN listing the reasons each query stage could not be executed
+natively. Nothing is logged when the entire stage runs in Comet. Useful as a
+low-noise check that fallback is or is not happening.
+
+### `spark.comet.logFallbackReasons.enabled`
+
+Logs every fallback reason as it is encountered, one WARN per reason. Use this
+when you want to see all reasons, including ones that
+`spark.comet.explainFallback.enabled` may aggregate or omit. The output does
+not include the surrounding plan, so it is best for accumulating diagnostics
+across many queries.
+
+### `spark.comet.explain.format`
+
+This config is read by `org.apache.comet.ExtendedExplainInfo`, which Spark
+loads via the `spark.sql.extendedExplainProviders` mechanism added in Spark
+4.0. Add the provider:
+
+```shell
+--conf spark.sql.extendedExplainProviders=org.apache.comet.ExtendedExplainInfo
+```
+
+The Spark SQL UI then shows an additional section under the detailed plan.
+The format is controlled by `spark.comet.explain.format`:
+
+- `verbose` (default): the full plan annotated with fallback reasons, plus a
+ summary of how much of the plan is accelerated.
+- `fallback`: a list of fallback reasons only.
+
+This is the most convenient option on Spark 4.0 because the output is shown
+inline in the UI. Earlier Spark versions do not have the
+`extendedExplainProviders` extension point, so this provider is not used and
+the config has no effect there.
+
+### `spark.comet.explain.native.enabled`
+
+When enabled, each executor task logs the DataFusion native plan it executes,
+along with metrics. This is verbose because there is one plan per task, but it
+is the only way to see the native plan as DataFusion sees it (including how
+operators were arranged after Comet's serialization). See the
+[Metrics Guide](metrics.md) for details on the native metrics that appear in
+this output.
+
+## Comet Operator Reference
+
+The following sections describe the Comet nodes you will see in plans, grouped
+by role. Names match what is shown in the plan output.
+
+### Scans
+
+| Node | Description |
+| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `CometScan` | V1 Parquet scan driven by Spark's file-source path through Comet's Parquet reader. Decoding runs in native code; the resulting Arrow batches cross JNI into the native plan. The active scan implementation is shown in brackets, e.g. `CometScan [native_iceberg_compat]`. |
+| `CometBatchScan` | DataSource V2 scan, including Iceberg Parquet, that produces Arrow batches consumed by Comet. |
+| `CometNativeScan` | Fully native Parquet scan that runs entirely in DataFusion (no JVM Parquet reader involvement). |
+| `CometIcebergNativeScan` | Fully native Iceberg Parquet scan. |
+| `CometCsvNativeScan` | Fully native CSV scan (experimental). |
+
+### Native Execution Operators
+
+These run natively in DataFusion. When several appear consecutively in a plan,
+they execute as a single fused native block.
+
+| Node | Spark equivalent |
+| ---------------------------- | ---------------------------------------------- |
+| `CometProject` | `ProjectExec` |
+| `CometFilter` | `FilterExec` |
+| `CometSort` | `SortExec` |
+| `CometLocalLimit` | `LocalLimitExec` |
+| `CometGlobalLimit` | `GlobalLimitExec` |
+| `CometExpand` | `ExpandExec` |
+| `CometExplode` | `GenerateExec` (for `explode` only) |
+| `CometHashAggregate` | `HashAggregateExec`, `ObjectHashAggregateExec` |
+| `CometHashJoin` | `ShuffledHashJoinExec` |
+| `CometBroadcastHashJoin` | `BroadcastHashJoinExec` |
+| `CometSortMergeJoin` | `SortMergeJoinExec` |
+| `CometWindow` | `WindowExec` |
+| `CometTakeOrderedAndProject` | `TakeOrderedAndProjectExec` |
+
+### JVM-Side Operators
+
+These keep their data on the JVM but participate in the Comet pipeline.
+
+| Node | Notes |
+| ------------------------ | ------------------------------------------------------------------------------------- |
+| `CometUnion` | JVM-side union of Comet inputs. The native side reads each branch as a separate scan. |
+| `CometCoalesce` | JVM-side partition coalesce. |
+| `CometCollectLimit` | JVM-side collect limit, equivalent to `CollectLimitExec`. |
+| `CometBroadcastExchange` | Broadcast exchange producing serialized Arrow batches that the consumer can decode. |
+| `CometSubqueryBroadcast` | Companion to `CometBroadcastExchange` for dynamic partition pruning subqueries. |
+
+### Shuffle Operators
+
+Comet has two shuffle implementations and the plan tells you which one is in
+use:
+
+- **`CometExchange`** is the **native shuffle** path. The child must already
+ be a Comet operator producing columnar Arrow batches; the node calls
+ `executeColumnar()` on its child and the partition, encode, and compress
+ steps run in native code. Hash and range partitioning **keys** must be
+ primitive types because native hashing and ordering do not support complex
+ types, but the data columns themselves can include `StructType`,
+ `ArrayType`, and `MapType` since batches are serialized via the Arrow IPC
+ writer.
+- **`CometColumnarExchange`** is the **JVM columnar shuffle** path. It accepts
+ either Spark row-based input or Comet columnar input, which makes it the
+ fallback when the child is not a Comet operator or when a hash/range key
+ type is not supported by native shuffle (for example, collated strings). It
+ is still preferred over Spark's native shuffle when Comet shuffle is
+ enabled.
+
+Both paths support the same set of partitioning schemes
+(`HashPartitioning`, `RangePartitioning`, `RoundRobinPartitioning`,
+`SinglePartition`) and both can carry complex types in data columns.
+
+The choice between the two is automatic. See the
+[Tuning Guide shuffle section](tuning.md#shuffle) for how to enable Comet
+shuffle and choose between the implementations.
+
+### Columnar/Row Transitions
+
+Comet inserts these nodes wherever data has to cross the columnar/row boundary.
+Multiple implementations exist because the optimal strategy depends on what
+produced the columnar data.
+
+| Node | Direction | Notes |
+| ------------------------------ | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `CometColumnarToRow` | columnar → row | JVM-based row conversion. A fork of Spark's `ColumnarToRowExec` that includes the SPARK-50235 fix. |
+| `CometNativeColumnarToRow` | columnar → row | Native row conversion that decodes broadcast Arrow batches via `NativeColumnarToRowConverter`. Used downstream of `CometBroadcastExchange`. Zero-copy for variable-length types and avoids an extra JVM materialization step. |
+| `CometSparkColumnarToColumnar` | columnar → columnar | Converts a Spark columnar input (a non-Comet `ColumnarBatch`) into Comet's Arrow batches. |
+| `CometSparkRowToColumnar` | row → columnar | Converts a Spark row input into Comet's Arrow batches. |
+
+The two `CometSpark*` names come from a single `CometSparkToColumnarExec`
+operator that picks the node name based on whether its child supports
+columnar.
diff --git a/_sources/user-guide/index.md.txt b/_sources/user-guide/index.md.txt
index 65736c1d4..7c603a75c 100644
--- a/_sources/user-guide/index.md.txt
+++ b/_sources/user-guide/index.md.txt
@@ -31,9 +31,9 @@ navigation menu to view its guide.
:caption: User Guides
:hidden:
-0.16.0-SNAPSHOT <latest/index>
+0.17.0-SNAPSHOT <latest/index>
+0.16.x <0.16/index>
0.15.x <0.15/index>
0.14.x <0.14/index>
0.13.x <0.13/index>
-0.12.x <0.12/index>
```
diff --git a/contributor-guide/index.html b/contributor-guide/index.html
index 25049493f..938578d68 100644
--- a/contributor-guide/index.html
+++ b/contributor-guide/index.html
@@ -66,7 +66,7 @@ under the License.
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Contributing to Apache DataFusion Comet" href="contributing.html" />
- <link rel="prev" title="Comet Kubernetes Support" href="../user-guide/0.12/kubernetes.html" />
+ <link rel="prev" title="Comet Kubernetes Support" href="../user-guide/0.13/kubernetes.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
@@ -481,7 +481,7 @@ menu to read more.</p>
<div class="prev-next-area">
<a class="left-prev"
- href="../user-guide/0.12/kubernetes.html"
+ href="../user-guide/0.13/kubernetes.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
diff --git a/objects.inv b/objects.inv
index 4790ce557..5aa7b6947 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/searchindex.js b/searchindex.js
index b402d0545..b7382f834 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"1. Format Your Code": [[14, "format-your-code"]], "1. Install Comet": [[16, "install-comet"], [24, "install-comet"]], "1. Native Operators (nativeExecs map)": [[3, "native-operators-nativeexecs-map"]], "2. Build and Verify": [[14, "build-and-verify"]], "2. Clone Iceberg and Apply Diff": [[16, "clone-iceberg-and-apply-diff"]], "2. Clone Spark and Apply Diff": [[24, "clone-spark-and-apply-diff"]], "2. Sink Operators (sinks map)": [[3, "sink-operators-sinks-m [...]
\ No newline at end of file
+Search.setIndex({"alltitles": {"1. Format Your Code": [[14, "format-your-code"]], "1. Install Comet": [[16, "install-comet"], [24, "install-comet"]], "1. Native Operators (nativeExecs map)": [[3, "native-operators-nativeexecs-map"]], "2. Build and Verify": [[14, "build-and-verify"]], "2. Clone Iceberg and Apply Diff": [[16, "clone-iceberg-and-apply-diff"]], "2. Clone Spark and Apply Diff": [[24, "clone-spark-and-apply-diff"]], "2. Sink Operators (sinks map)": [[3, "sink-operators-sinks-m [...]
\ No newline at end of file
diff --git a/user-guide/0.12/compatibility.html b/user-guide/0.12/compatibility.html
deleted file mode 100644
index 239669327..000000000
--- a/user-guide/0.12/compatibility.html
+++ /dev/null
@@ -1,959 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-
-<!DOCTYPE html>
-
-
-<html lang="en" data-content_root="../../" data-theme="light">
-
- <head>
- <meta charset="utf-8" />
- <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
-
- <title>Compatibility Guide — Apache DataFusion Comet documentation</title>
-
-
-
- <script data-cfasync="false">
- document.documentElement.dataset.mode = localStorage.getItem("mode") || "light";
- document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
- </script>
- <!--
- this give us a css class that will be invisible only if js is disabled
- -->
- <noscript>
- <style>
- .pst-js-only { display: none !important; }
-
- </style>
- </noscript>
-
- <!-- Loaded before other Sphinx assets -->
- <link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-
- <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=8f2a1f02" />
- <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=cd442bcd" />
-
- <!-- So that users can add custom icons -->
- <script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
- <!-- Pre-loaded scripts that we'll load fully later -->
- <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
-<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
-
- <script src="../../_static/documentation_options.js?v=5929fcd5"></script>
- <script src="../../_static/doctools.js?v=9a2dae69"></script>
- <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
- <script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/0.12/compatibility';</script>
- <script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
- <link rel="index" title="Index" href="../../genindex.html" />
- <link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Comet Tuning Guide" href="tuning.html" />
- <link rel="prev" title="Comet Configuration Settings" href="configs.html" />
- <meta name="viewport" content="width=device-width, initial-scale=1"/>
- <meta name="docsearch:language" content="en"/>
- <meta name="docsearch:version" content="" />
- </head>
-
-
- <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="light">
-
-
-
- <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
-
- <div id="pst-scroll-pixel-helper"></div>
-
- <button type="button" class="btn rounded-pill" id="pst-back-to-top">
- <i class="fa-solid fa-arrow-up"></i>Back to top</button>
-
-
- <dialog id="pst-search-dialog">
-
-<form class="bd-search d-flex align-items-center"
- action="../../search.html"
- method="get">
- <i class="fa-solid fa-magnifying-glass"></i>
- <input type="search"
- class="form-control"
- name="q"
- placeholder="Search the docs ..."
- aria-label="Search the docs ..."
- autocomplete="off"
- autocorrect="off"
- autocapitalize="off"
- spellcheck="false"/>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
-</form>
- </dialog>
-
- <div class="pst-async-banner-revealer d-none">
- <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
-</div>
-
-
- <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
-<div class="bd-header__inner bd-page-width">
- <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
- <span class="fa-solid fa-bars"></span>
- </button>
-
-
- <div class="col-lg-3 navbar-header-items__start">
-
- <div class="navbar-item">
-
-
-
-
-
-<a class="navbar-brand logo" href="../../index.html">
-
-
-
-
-
-
-
-
- <img src="../../_static/DataFusionComet-Logo-Light.png" class="logo__image only-light" alt="Apache DataFusion Comet documentation - Home"/>
- <img src="../../_static/DataFusionComet-Logo-Dark.png" class="logo__image only-dark pst-js-only" alt="Apache DataFusion Comet documentation - Home"/>
-
-
-</a></div>
-
- </div>
-
- <div class="col-lg-9 navbar-header-items">
-
- <div class="me-auto navbar-header-items__center">
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
- </div>
-
-
- <div class="navbar-header-items__end">
-
- <div class="navbar-item navbar-persistent--container">
-
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
-
- <div class="navbar-persistent--mobile">
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
-
-</div>
-
- </header>
-
-
- <div class="bd-container">
- <div class="bd-container__inner bd-page-width">
-
-
-
- <dialog id="pst-primary-sidebar-modal"></dialog>
- <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
-
-
-
- <div class="sidebar-header-items sidebar-primary__section">
-
-
- <div class="sidebar-header-items__center">
-
-
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
-
- </div>
-
-
-
- <div class="sidebar-header-items__end">
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
- <div class="sidebar-primary-items__start sidebar-primary__section">
- <div class="sidebar-primary-item"><!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
- <div class="bd-toc-item active">
- <p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
-<ul class="current">
-<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.12.x</a><ul class="current">
-<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
-<li class="toctree-l3"><a class="reference internal" href="source.html">Building From Source</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datasources.html">Supported Data Sources</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datatypes.html">Supported Data Types</a></li>
-<li class="toctree-l3"><a class="reference internal" href="operators.html">Supported Operators</a></li>
-<li class="toctree-l3"><a class="reference internal" href="expressions.html">Supported Expressions</a></li>
-<li class="toctree-l3"><a class="reference internal" href="configs.html">Configuration Settings</a></li>
-<li class="toctree-l3 current"><a class="current reference internal" href="#">Compatibility Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="tuning.html">Tuning Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="metrics.html">Metrics Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
-</ul>
-</li>
-</ul>
-</li>
-<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../about/gluten_comparison.html">Comparison with Gluten</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../asf/index.html">ASF Links</a></li>
-</ul>
-
- </div>
-</nav>
-</div>
- </div>
-
-
- <div class="sidebar-primary-items__end sidebar-primary__section">
- <div class="sidebar-primary-item">
-<div id="ethical-ad-placement"
- class="flat"
- data-ea-publisher="readthedocs"
- data-ea-type="readthedocs-sidebar"
- data-ea-manual="true">
-</div></div>
- </div>
-
-
- </div>
-
- <main id="main-content" class="bd-main" role="main">
-
-
- <div class="bd-content">
- <div class="bd-article-container">
-
- <div class="bd-header-article d-print-none">
-<div class="header-article-items header-article__inner">
-
- <div class="header-article-items__start">
-
- <div class="header-article-item">
-
-<nav aria-label="Breadcrumb" class="d-print-none">
- <ul class="bd-breadcrumbs">
-
- <li class="breadcrumb-item breadcrumb-home">
- <a href="../../index.html" class="nav-link" aria-label="Home">
- <i class="fa-solid fa-home"></i>
- </a>
- </li>
-
- <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Comet User Guide</a></li>
-
-
- <li class="breadcrumb-item"><a href="index.html" class="nav-link">Comet 0.12.0 User Guide</a></li>
-
- <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Compatibility Guide</span></li>
- </ul>
-</nav>
-</div>
-
- </div>
-
-
-</div>
-</div>
-
-
-
-
-<div id="searchbox"></div>
- <article class="bd-article">
-
- <!---
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<div class="admonition warning">
-<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
-</div>
-<section id="compatibility-guide">
-<h1>Compatibility Guide<a class="headerlink" href="#compatibility-guide" title="Link to this heading">#</a></h1>
-<p>Comet aims to provide consistent results with the version of Apache Spark that is being used.</p>
-<p>This guide offers information about areas of functionality where there are known differences.</p>
-<section id="parquet">
-<h2>Parquet<a class="headerlink" href="#parquet" title="Link to this heading">#</a></h2>
-<p>Comet has the following limitations when reading Parquet files:</p>
-<ul class="simple">
-<li><p>Comet does not support reading decimals encoded in binary format.</p></li>
-<li><p>No support for default values that are nested types (e.g., maps, arrays, structs). Literal default values are supported.</p></li>
-</ul>
-</section>
-<section id="ansi-mode">
-<h2>ANSI Mode<a class="headerlink" href="#ansi-mode" title="Link to this heading">#</a></h2>
-<p>Comet will fall back to Spark for the following expressions when ANSI mode is enabled. Thes expressions can be enabled by setting
-<code class="docutils literal notranslate"><span class="pre">spark.comet.expression.EXPRNAME.allowIncompatible=true</span></code>, where <code class="docutils literal notranslate"><span class="pre">EXPRNAME</span></code> is the Spark expression class name. See
-the <a class="reference internal" href="expressions.html"><span class="std std-doc">Comet Supported Expressions Guide</span></a> for more information on this configuration setting.</p>
-<ul class="simple">
-<li><p>Average</p></li>
-<li><p>Sum</p></li>
-<li><p>Cast (in some cases)</p></li>
-</ul>
-<p>There is an <a class="reference external" href="https://github.com/apache/datafusion-comet/issues/313">epic</a> where we are tracking the work to fully implement ANSI support.</p>
-</section>
-<section id="floating-point-number-comparison">
-<h2>Floating-point Number Comparison<a class="headerlink" href="#floating-point-number-comparison" title="Link to this heading">#</a></h2>
-<p>Spark normalizes NaN and zero for floating point numbers for several cases. See <code class="docutils literal notranslate"><span class="pre">NormalizeFloatingNumbers</span></code> optimization rule in Spark.
-However, one exception is comparison. Spark does not normalize NaN and zero when comparing values
-because they are handled well in Spark (e.g., <code class="docutils literal notranslate"><span class="pre">SQLOrderingUtil.compareFloats</span></code>). But the comparison
-functions of arrow-rs used by DataFusion do not normalize NaN and zero (e.g., <a class="reference external" href="https://docs.rs/arrow/latest/arrow/compute/kernels/cmp/fn.eq.html#">arrow::compute::kernels::cmp::eq</a>).
-So Comet adds additional normalization expression of NaN and zero for comparisons, and may still have differences
-to Spark in some cases, especially when the data contains both positive and negative zero. This is likely an edge
-case that is not of concern for many users. If it is a concern, setting <code class="docutils literal notranslate"><span class="pre">spark.comet.exec.strictFloatingPoint=true</span></code>
-will make relevant operations fall back to Spark.</p>
-</section>
-<section id="incompatible-expressions">
-<h2>Incompatible Expressions<a class="headerlink" href="#incompatible-expressions" title="Link to this heading">#</a></h2>
-<p>Expressions that are not 100% Spark-compatible will fall back to Spark by default and can be enabled by setting
-<code class="docutils literal notranslate"><span class="pre">spark.comet.expression.EXPRNAME.allowIncompatible=true</span></code>, where <code class="docutils literal notranslate"><span class="pre">EXPRNAME</span></code> is the Spark expression class name. See
-the <a class="reference internal" href="expressions.html"><span class="std std-doc">Comet Supported Expressions Guide</span></a> for more information on this configuration setting.</p>
-</section>
-<section id="regular-expressions">
-<h2>Regular Expressions<a class="headerlink" href="#regular-expressions" title="Link to this heading">#</a></h2>
-<p>Comet uses the Rust regexp crate for evaluating regular expressions, and this has different behavior from Java’s
-regular expression engine. Comet will fall back to Spark for patterns that are known to produce different results, but
-this can be overridden by setting <code class="docutils literal notranslate"><span class="pre">spark.comet.regexp.allowIncompatible=true</span></code>.</p>
-</section>
-<section id="window-functions">
-<h2>Window Functions<a class="headerlink" href="#window-functions" title="Link to this heading">#</a></h2>
-<p>Comet’s support for window functions is incomplete and known to be incorrect. It is disabled by default and
-should not be used in production. The feature will be enabled in a future release. Tracking issue: <a class="reference external" href="https://github.com/apache/datafusion-comet/issues/2721">#2721</a>.</p>
-</section>
-<section id="cast">
-<h2>Cast<a class="headerlink" href="#cast" title="Link to this heading">#</a></h2>
-<p>Cast operations in Comet fall into three levels of support:</p>
-<ul class="simple">
-<li><p><strong>Compatible</strong>: The results match Apache Spark</p></li>
-<li><p><strong>Incompatible</strong>: The results may match Apache Spark for some inputs, but there are known issues where some inputs
-will result in incorrect results or exceptions. The query stage will fall back to Spark by default. Setting
-<code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Cast.allowIncompatible=true</span></code> will allow all incompatible casts to run natively in Comet, but this is not
-recommended for production use.</p></li>
-<li><p><strong>Unsupported</strong>: Comet does not provide a native version of this cast expression and the query stage will fall back to
-Spark.</p></li>
-</ul>
-<section id="compatible-casts">
-<h3>Compatible Casts<a class="headerlink" href="#compatible-casts" title="Link to this heading">#</a></h3>
-<p>The following cast operations are generally compatible with Spark except for the differences noted here.</p>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:COMPAT_CAST_TABLE-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>From Type</p></th>
-<th class="head"><p>To Type</p></th>
-<th class="head"><p>Notes</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p>boolean</p></td>
-<td><p>byte</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>boolean</p></td>
-<td><p>short</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>boolean</p></td>
-<td><p>integer</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>boolean</p></td>
-<td><p>long</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>boolean</p></td>
-<td><p>float</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>boolean</p></td>
-<td><p>double</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>boolean</p></td>
-<td><p>string</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>byte</p></td>
-<td><p>boolean</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>byte</p></td>
-<td><p>short</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>byte</p></td>
-<td><p>integer</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>byte</p></td>
-<td><p>long</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>byte</p></td>
-<td><p>float</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>byte</p></td>
-<td><p>double</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>byte</p></td>
-<td><p>decimal</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>byte</p></td>
-<td><p>string</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>short</p></td>
-<td><p>boolean</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>short</p></td>
-<td><p>byte</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>short</p></td>
-<td><p>integer</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>short</p></td>
-<td><p>long</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>short</p></td>
-<td><p>float</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>short</p></td>
-<td><p>double</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>short</p></td>
-<td><p>decimal</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>short</p></td>
-<td><p>string</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>integer</p></td>
-<td><p>boolean</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>integer</p></td>
-<td><p>byte</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>integer</p></td>
-<td><p>short</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>integer</p></td>
-<td><p>long</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>integer</p></td>
-<td><p>float</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>integer</p></td>
-<td><p>double</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>integer</p></td>
-<td><p>decimal</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>integer</p></td>
-<td><p>string</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>long</p></td>
-<td><p>boolean</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>long</p></td>
-<td><p>byte</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>long</p></td>
-<td><p>short</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>long</p></td>
-<td><p>integer</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>long</p></td>
-<td><p>float</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>long</p></td>
-<td><p>double</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>long</p></td>
-<td><p>decimal</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>long</p></td>
-<td><p>string</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>float</p></td>
-<td><p>boolean</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>float</p></td>
-<td><p>byte</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>float</p></td>
-<td><p>short</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>float</p></td>
-<td><p>integer</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>float</p></td>
-<td><p>long</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>float</p></td>
-<td><p>double</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>float</p></td>
-<td><p>string</p></td>
-<td><p>There can be differences in precision. For example, the input “1.4E-45” will produce 1.0E-45 instead of 1.4E-45</p></td>
-</tr>
-<tr class="row-even"><td><p>double</p></td>
-<td><p>boolean</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>double</p></td>
-<td><p>byte</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>double</p></td>
-<td><p>short</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>double</p></td>
-<td><p>integer</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>double</p></td>
-<td><p>long</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>double</p></td>
-<td><p>float</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>double</p></td>
-<td><p>string</p></td>
-<td><p>There can be differences in precision. For example, the input “1.4E-45” will produce 1.0E-45 instead of 1.4E-45</p></td>
-</tr>
-<tr class="row-odd"><td><p>decimal</p></td>
-<td><p>boolean</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>decimal</p></td>
-<td><p>byte</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>decimal</p></td>
-<td><p>short</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>decimal</p></td>
-<td><p>integer</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>decimal</p></td>
-<td><p>long</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>decimal</p></td>
-<td><p>float</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>decimal</p></td>
-<td><p>double</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>decimal</p></td>
-<td><p>decimal</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>decimal</p></td>
-<td><p>string</p></td>
-<td><p>There can be formatting differences in some case due to Spark using scientific notation where Comet does not</p></td>
-</tr>
-<tr class="row-even"><td><p>string</p></td>
-<td><p>boolean</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>string</p></td>
-<td><p>byte</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>string</p></td>
-<td><p>short</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>string</p></td>
-<td><p>integer</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>string</p></td>
-<td><p>long</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>string</p></td>
-<td><p>binary</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>string</p></td>
-<td><p>date</p></td>
-<td><p>Only supports years between 262143 BC and 262142 AD</p></td>
-</tr>
-<tr class="row-odd"><td><p>binary</p></td>
-<td><p>string</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>date</p></td>
-<td><p>string</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>timestamp</p></td>
-<td><p>long</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>timestamp</p></td>
-<td><p>string</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>timestamp</p></td>
-<td><p>date</p></td>
-<td><p></p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:COMPAT_CAST_TABLE-->
-</section>
-<section id="incompatible-casts">
-<h3>Incompatible Casts<a class="headerlink" href="#incompatible-casts" title="Link to this heading">#</a></h3>
-<p>The following cast operations are not compatible with Spark for all inputs and are disabled by default.</p>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:INCOMPAT_CAST_TABLE-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>From Type</p></th>
-<th class="head"><p>To Type</p></th>
-<th class="head"><p>Notes</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p>float</p></td>
-<td><p>decimal</p></td>
-<td><p>There can be rounding differences</p></td>
-</tr>
-<tr class="row-odd"><td><p>double</p></td>
-<td><p>decimal</p></td>
-<td><p>There can be rounding differences</p></td>
-</tr>
-<tr class="row-even"><td><p>string</p></td>
-<td><p>float</p></td>
-<td><p>Does not support inputs ending with ‘d’ or ‘f’. Does not support ‘inf’. Does not support ANSI mode.</p></td>
-</tr>
-<tr class="row-odd"><td><p>string</p></td>
-<td><p>double</p></td>
-<td><p>Does not support inputs ending with ‘d’ or ‘f’. Does not support ‘inf’. Does not support ANSI mode.</p></td>
-</tr>
-<tr class="row-even"><td><p>string</p></td>
-<td><p>decimal</p></td>
-<td><p>Does not support inputs ending with ‘d’ or ‘f’. Does not support ‘inf’. Does not support ANSI mode. Returns 0.0 instead of null if input contains no digits</p></td>
-</tr>
-<tr class="row-odd"><td><p>string</p></td>
-<td><p>timestamp</p></td>
-<td><p>Not all valid formats are supported</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:INCOMPAT_CAST_TABLE-->
-</section>
-<section id="unsupported-casts">
-<h3>Unsupported Casts<a class="headerlink" href="#unsupported-casts" title="Link to this heading">#</a></h3>
-<p>Any cast not listed in the previous tables is currently unsupported. We are working on adding more. See the
-<a class="reference external" href="https://github.com/apache/datafusion-comet/issues/286">tracking issue</a> for more details.</p>
-</section>
-</section>
-</section>
-
-
- </article>
-
-
-
-
-
- <footer class="prev-next-footer d-print-none">
-
-<div class="prev-next-area">
- <a class="left-prev"
- href="configs.html"
- title="previous page">
- <i class="fa-solid fa-angle-left"></i>
- <div class="prev-next-info">
- <p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Comet Configuration Settings</p>
- </div>
- </a>
- <a class="right-next"
- href="tuning.html"
- title="next page">
- <div class="prev-next-info">
- <p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Comet Tuning Guide</p>
- </div>
- <i class="fa-solid fa-angle-right"></i>
- </a>
-</div>
- </footer>
-
- </div>
-
-
-
-
- </div>
- <footer class="bd-footer-content">
-
- </footer>
-
- </main>
- </div>
- </div>
-
- <!-- Scripts loaded after <body> so the DOM is not blocked -->
- <script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
-<script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
-
-<!-- Based on pydata_sphinx_theme/footer.html -->
-<footer class="footer mt-5 mt-md-0">
- <div class="container">
-
- <div class="footer-item">
- <p>Apache DataFusion, Apache DataFusion Comet, Apache, the Apache feather logo, and the Apache DataFusion project logo</p>
- <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
- </div>
- </div>
-</footer>
-
-
- </body>
-</html>
\ No newline at end of file
diff --git a/user-guide/0.12/configs.html b/user-guide/0.12/configs.html
deleted file mode 100644
index 6bc294aff..000000000
--- a/user-guide/0.12/configs.html
+++ /dev/null
@@ -1,1691 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-
-<!DOCTYPE html>
-
-
-<html lang="en" data-content_root="../../" data-theme="light">
-
- <head>
- <meta charset="utf-8" />
- <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
-
- <title>Comet Configuration Settings — Apache DataFusion Comet documentation</title>
-
-
-
- <script data-cfasync="false">
- document.documentElement.dataset.mode = localStorage.getItem("mode") || "light";
- document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
- </script>
- <!--
- this give us a css class that will be invisible only if js is disabled
- -->
- <noscript>
- <style>
- .pst-js-only { display: none !important; }
-
- </style>
- </noscript>
-
- <!-- Loaded before other Sphinx assets -->
- <link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-
- <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=8f2a1f02" />
- <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=cd442bcd" />
-
- <!-- So that users can add custom icons -->
- <script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
- <!-- Pre-loaded scripts that we'll load fully later -->
- <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
-<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
-
- <script src="../../_static/documentation_options.js?v=5929fcd5"></script>
- <script src="../../_static/doctools.js?v=9a2dae69"></script>
- <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
- <script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/0.12/configs';</script>
- <script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
- <link rel="index" title="Index" href="../../genindex.html" />
- <link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Compatibility Guide" href="compatibility.html" />
- <link rel="prev" title="Supported Spark Expressions" href="expressions.html" />
- <meta name="viewport" content="width=device-width, initial-scale=1"/>
- <meta name="docsearch:language" content="en"/>
- <meta name="docsearch:version" content="" />
- </head>
-
-
- <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="light">
-
-
-
- <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
-
- <div id="pst-scroll-pixel-helper"></div>
-
- <button type="button" class="btn rounded-pill" id="pst-back-to-top">
- <i class="fa-solid fa-arrow-up"></i>Back to top</button>
-
-
- <dialog id="pst-search-dialog">
-
-<form class="bd-search d-flex align-items-center"
- action="../../search.html"
- method="get">
- <i class="fa-solid fa-magnifying-glass"></i>
- <input type="search"
- class="form-control"
- name="q"
- placeholder="Search the docs ..."
- aria-label="Search the docs ..."
- autocomplete="off"
- autocorrect="off"
- autocapitalize="off"
- spellcheck="false"/>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
-</form>
- </dialog>
-
- <div class="pst-async-banner-revealer d-none">
- <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
-</div>
-
-
- <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
-<div class="bd-header__inner bd-page-width">
- <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
- <span class="fa-solid fa-bars"></span>
- </button>
-
-
- <div class="col-lg-3 navbar-header-items__start">
-
- <div class="navbar-item">
-
-
-
-
-
-<a class="navbar-brand logo" href="../../index.html">
-
-
-
-
-
-
-
-
- <img src="../../_static/DataFusionComet-Logo-Light.png" class="logo__image only-light" alt="Apache DataFusion Comet documentation - Home"/>
- <img src="../../_static/DataFusionComet-Logo-Dark.png" class="logo__image only-dark pst-js-only" alt="Apache DataFusion Comet documentation - Home"/>
-
-
-</a></div>
-
- </div>
-
- <div class="col-lg-9 navbar-header-items">
-
- <div class="me-auto navbar-header-items__center">
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
- </div>
-
-
- <div class="navbar-header-items__end">
-
- <div class="navbar-item navbar-persistent--container">
-
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
-
- <div class="navbar-persistent--mobile">
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
-
-</div>
-
- </header>
-
-
- <div class="bd-container">
- <div class="bd-container__inner bd-page-width">
-
-
-
- <dialog id="pst-primary-sidebar-modal"></dialog>
- <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
-
-
-
- <div class="sidebar-header-items sidebar-primary__section">
-
-
- <div class="sidebar-header-items__center">
-
-
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
-
- </div>
-
-
-
- <div class="sidebar-header-items__end">
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
- <div class="sidebar-primary-items__start sidebar-primary__section">
- <div class="sidebar-primary-item"><!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
- <div class="bd-toc-item active">
- <p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
-<ul class="current">
-<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.12.x</a><ul class="current">
-<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
-<li class="toctree-l3"><a class="reference internal" href="source.html">Building From Source</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datasources.html">Supported Data Sources</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datatypes.html">Supported Data Types</a></li>
-<li class="toctree-l3"><a class="reference internal" href="operators.html">Supported Operators</a></li>
-<li class="toctree-l3"><a class="reference internal" href="expressions.html">Supported Expressions</a></li>
-<li class="toctree-l3 current"><a class="current reference internal" href="#">Configuration Settings</a></li>
-<li class="toctree-l3"><a class="reference internal" href="compatibility.html">Compatibility Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="tuning.html">Tuning Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="metrics.html">Metrics Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
-</ul>
-</li>
-</ul>
-</li>
-<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../about/gluten_comparison.html">Comparison with Gluten</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../asf/index.html">ASF Links</a></li>
-</ul>
-
- </div>
-</nav>
-</div>
- </div>
-
-
- <div class="sidebar-primary-items__end sidebar-primary__section">
- <div class="sidebar-primary-item">
-<div id="ethical-ad-placement"
- class="flat"
- data-ea-publisher="readthedocs"
- data-ea-type="readthedocs-sidebar"
- data-ea-manual="true">
-</div></div>
- </div>
-
-
- </div>
-
- <main id="main-content" class="bd-main" role="main">
-
-
- <div class="bd-content">
- <div class="bd-article-container">
-
- <div class="bd-header-article d-print-none">
-<div class="header-article-items header-article__inner">
-
- <div class="header-article-items__start">
-
- <div class="header-article-item">
-
-<nav aria-label="Breadcrumb" class="d-print-none">
- <ul class="bd-breadcrumbs">
-
- <li class="breadcrumb-item breadcrumb-home">
- <a href="../../index.html" class="nav-link" aria-label="Home">
- <i class="fa-solid fa-home"></i>
- </a>
- </li>
-
- <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Comet User Guide</a></li>
-
-
- <li class="breadcrumb-item"><a href="index.html" class="nav-link">Comet 0.12.0 User Guide</a></li>
-
- <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Comet Configuration Settings</span></li>
- </ul>
-</nav>
-</div>
-
- </div>
-
-
-</div>
-</div>
-
-
-
-
-<div id="searchbox"></div>
- <article class="bd-article">
-
- <!---
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<div class="admonition warning">
-<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
-</div>
-<section id="comet-configuration-settings">
-<h1>Comet Configuration Settings<a class="headerlink" href="#comet-configuration-settings" title="Link to this heading">#</a></h1>
-<p>Comet provides the following configuration settings.</p>
-<section id="scan-configuration-settings">
-<h2>Scan Configuration Settings<a class="headerlink" href="#scan-configuration-settings" title="Link to this heading">#</a></h2>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[scan]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.scan.allowIncompatible</span></code></p></td>
-<td><p>Some Comet scan implementations are not currently fully compatible with Spark for all datatypes. Set this config to true to allow them anyway. For more information, refer to the <a class="reference external" href="https://datafusion.apache.org/comet/user-guide/compatibility.html">Comet Compatibility Guide</a>.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.scan.enabled</span></code></p></td>
-<td><p>Whether to enable native scans. When this is turned on, Spark will use Comet to read supported data sources (currently only Parquet is supported natively). Note that to enable native vectorized execution, both this config and <code class="docutils literal notranslate"><span class="pre">spark.comet.exec.enabled</span></code> need to be enabled.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.scan.icebergNative.enabled</span></code></p></td>
-<td><p>Whether to enable native Iceberg table scan using iceberg-rust. When enabled, Iceberg tables are read directly through native execution, bypassing Spark’s DataSource V2 API for better performance.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.scan.preFetch.enabled</span></code></p></td>
-<td><p>Whether to enable pre-fetching feature of CometScan.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.scan.preFetch.threadNum</span></code></p></td>
-<td><p>The number of threads running pre-fetching for CometScan. Effective if spark.comet.scan.preFetch.enabled is enabled. Note that more pre-fetching threads means more memory requirement to store pre-fetched row groups.</p></td>
-<td><p>2</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.hadoop.fs.comet.libhdfs.schemes</span></code></p></td>
-<td><p>Defines filesystem schemes (e.g., hdfs, webhdfs) that the native side accesses via libhdfs, separated by commas. Valid only when built with hdfs feature enabled.</p></td>
-<td><p></p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-<section id="parquet-reader-configuration-settings">
-<h2>Parquet Reader Configuration Settings<a class="headerlink" href="#parquet-reader-configuration-settings" title="Link to this heading">#</a></h2>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[parquet]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.parquet.enable.directBuffer</span></code></p></td>
-<td><p>Whether to use Java direct byte buffer when reading Parquet.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.parquet.read.io.adjust.readRange.skew</span></code></p></td>
-<td><p>In the parallel reader, if the read ranges submitted are skewed in sizes, this option will cause the reader to break up larger read ranges into smaller ranges to reduce the skew. This will result in a slightly larger number of connections opened to the file system but may give improved performance.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.parquet.read.io.mergeRanges</span></code></p></td>
-<td><p>When enabled the parallel reader will try to merge ranges of data that are separated by less than <code class="docutils literal notranslate"><span class="pre">comet.parquet.read.io.mergeRanges.delta</span></code> bytes. Longer continuous reads are faster on cloud storage.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.parquet.read.io.mergeRanges.delta</span></code></p></td>
-<td><p>The delta in bytes between consecutive read ranges below which the parallel reader will try to merge the ranges. The default is 8MB.</p></td>
-<td><p>8388608</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.parquet.read.parallel.io.enabled</span></code></p></td>
-<td><p>Whether to enable Comet’s parallel reader for Parquet files. The parallel reader reads ranges of consecutive data in a file in parallel. It is faster for large files and row groups but uses more resources.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.parquet.read.parallel.io.thread-pool.size</span></code></p></td>
-<td><p>The maximum number of parallel threads the parallel reader will use in a single executor. For executors configured with a smaller number of cores, use a smaller number.</p></td>
-<td><p>16</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.parquet.respectFilterPushdown</span></code></p></td>
-<td><p>Whether to respect Spark’s PARQUET_FILTER_PUSHDOWN_ENABLED config. This needs to be respected when running the Spark SQL test suite but the default setting results in poor performance in Comet when using the new native scans, disabled by default</p></td>
-<td><p>false</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-<section id="query-execution-settings">
-<h2>Query Execution Settings<a class="headerlink" href="#query-execution-settings" title="Link to this heading">#</a></h2>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[exec]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.caseConversion.enabled</span></code></p></td>
-<td><p>Java uses locale-specific rules when converting strings to upper or lower case and Rust does not, so we disable upper and lower by default.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.debug.enabled</span></code></p></td>
-<td><p>Whether to enable debug mode for Comet. When enabled, Comet will do additional checks for debugging purpose. For example, validating array when importing arrays from JVM at native side. Note that these checks may be expensive in performance and should only be enabled for debugging purpose.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.dppFallback.enabled</span></code></p></td>
-<td><p>Whether to fall back to Spark for queries that use DPP.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.enabled</span></code></p></td>
-<td><p>Whether to enable Comet extension for Spark. When this is turned on, Spark will use Comet to read Parquet data source. Note that to enable native vectorized execution, both this config and <code class="docutils literal notranslate"><span class="pre">spark.comet.exec.enabled</span></code> need to be enabled. It can be overridden by the environment variable <code class="docutils literal notranslate"><span class="pre">ENABLE_COMET</span></code>.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exceptionOnDatetimeRebase</span></code></p></td>
-<td><p>Whether to throw exception when seeing dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar. Since Spark 3, dates/timestamps were written according to the Proleptic Gregorian calendar. When this is true, Comet will throw exceptions when seeing these dates/timestamps that were written by Spark version before 3.0. If this is false, these dates/timestamps will be read as if they were written to the Proleptic Gregorian calendar and will not be rebased.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.enabled</span></code></p></td>
-<td><p>Whether to enable Comet native vectorized execution for Spark. This controls whether Spark should convert operators into their Comet counterparts and execute them in native space. Note: each operator is associated with a separate config in the format of <code class="docutils literal notranslate"><span class="pre">spark.comet.exec.<operator_name>.enabled</span></code> at the moment, and both the config and this need to be turned on, in order for the operator to be executed in [...]
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.replaceSortMergeJoin</span></code></p></td>
-<td><p>Experimental feature to force Spark to replace SortMergeJoin with ShuffledHashJoin for improved performance. This feature is not stable yet. For more information, refer to the <a class="reference external" href="https://datafusion.apache.org/comet/user-guide/tuning.html">Comet Tuning Guide</a>.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.strictFloatingPoint</span></code></p></td>
-<td><p>When enabled, fall back to Spark for floating-point operations that may differ from Spark, such as when comparing or sorting -0.0 and 0.0. For more information, refer to the <a class="reference external" href="https://datafusion.apache.org/comet/user-guide/compatibility.html">Comet Compatibility Guide</a>.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.maxTempDirectorySize</span></code></p></td>
-<td><p>The maximum amount of data (in bytes) stored inside the temporary directories.</p></td>
-<td><p>107374182400b</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.metrics.updateInterval</span></code></p></td>
-<td><p>The interval in milliseconds to update metrics. If interval is negative, metrics will be updated upon task completion.</p></td>
-<td><p>3000</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.nativeLoadRequired</span></code></p></td>
-<td><p>Whether to require Comet native library to load successfully when Comet is enabled. If not, Comet will silently fallback to Spark when it fails to load the native lib. Otherwise, an error will be thrown and the Spark job will be aborted.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.regexp.allowIncompatible</span></code></p></td>
-<td><p>Comet is not currently fully compatible with Spark for all regular expressions. Set this config to true to allow them anyway. For more information, refer to the <a class="reference external" href="https://datafusion.apache.org/comet/user-guide/compatibility.html">Comet Compatibility Guide</a>.</p></td>
-<td><p>false</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-<section id="viewing-explain-plan-fallback-reasons">
-<h2>Viewing Explain Plan & Fallback Reasons<a class="headerlink" href="#viewing-explain-plan-fallback-reasons" title="Link to this heading">#</a></h2>
-<p>These settings can be used to determine which parts of the plan are accelerated by Comet and to see why some parts of the plan could not be supported by Comet.</p>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[exec_explain]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.explain.format</span></code></p></td>
-<td><p>Choose extended explain output. The default format of ‘verbose’ will provide the full query plan annotated with fallback reasons as well as a summary of how much of the plan was accelerated by Comet. The format ‘fallback’ provides a list of fallback reasons instead.</p></td>
-<td><p>verbose</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.explain.native.enabled</span></code></p></td>
-<td><p>When this setting is enabled, Comet will provide a tree representation of the native query plan before execution and again after execution, with metrics.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.explain.rules</span></code></p></td>
-<td><p>When this setting is enabled, Comet will log all plan transformations performed in physical optimizer rules. Default: false</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.explainFallback.enabled</span></code></p></td>
-<td><p>When this setting is enabled, Comet will provide logging explaining the reason(s) why a query stage cannot be executed natively. Set this to false to reduce the amount of logging.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.logFallbackReasons.enabled</span></code></p></td>
-<td><p>When this setting is enabled, Comet will log warnings for all fallback reasons. It can be overridden by the environment variable <code class="docutils literal notranslate"><span class="pre">ENABLE_COMET_LOG_FALLBACK_REASONS</span></code>.</p></td>
-<td><p>false</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-<section id="shuffle-configuration-settings">
-<h2>Shuffle Configuration Settings<a class="headerlink" href="#shuffle-configuration-settings" title="Link to this heading">#</a></h2>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[shuffle]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.async.enabled</span></code></p></td>
-<td><p>Whether to enable asynchronous shuffle for Arrow-based shuffle.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.async.max.thread.num</span></code></p></td>
-<td><p>Maximum number of threads on an executor used for Comet async columnar shuffle. This is the upper bound of total number of shuffle threads per executor. In other words, if the number of cores * the number of shuffle threads per task <code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.async.thread.num</span></code> is larger than this config. Comet will use this config as the number of shuffle threads per executor instead.</p></td>
-<td><p>100</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.async.thread.num</span></code></p></td>
-<td><p>Number of threads used for Comet async columnar shuffle per shuffle task. Note that more threads means more memory requirement to buffer shuffle data before flushing to disk. Also, more threads may not always improve performance, and should be set based on the number of cores available.</p></td>
-<td><p>3</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.batch.size</span></code></p></td>
-<td><p>Batch size when writing out sorted spill files on the native side. Note that this should not be larger than batch size (i.e., <code class="docutils literal notranslate"><span class="pre">spark.comet.batchSize</span></code>). Otherwise it will produce larger batches than expected in the native operator after shuffle.</p></td>
-<td><p>8192</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.shuffle.compression.codec</span></code></p></td>
-<td><p>The codec of Comet native shuffle used to compress shuffle data. lz4, zstd, and snappy are supported. Compression can be disabled by setting spark.shuffle.compress=false.</p></td>
-<td><p>lz4</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.shuffle.compression.zstd.level</span></code></p></td>
-<td><p>The compression level to use when compressing shuffle files with zstd.</p></td>
-<td><p>1</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.shuffle.enabled</span></code></p></td>
-<td><p>Whether to enable Comet native shuffle. Note that this requires setting <code class="docutils literal notranslate"><span class="pre">spark.shuffle.manager</span></code> to <code class="docutils literal notranslate"><span class="pre">org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager</span></code>. <code class="docutils literal notranslate"><span class="pre">spark.shuffle.manager</span></code> must be set before starting the Spark application and cannot be changed dur [...]
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.native.shuffle.partitioning.hash.enabled</span></code></p></td>
-<td><p>Whether to enable hash partitioning for Comet native shuffle.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.native.shuffle.partitioning.range.enabled</span></code></p></td>
-<td><p>Whether to enable range partitioning for Comet native shuffle.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.shuffle.preferDictionary.ratio</span></code></p></td>
-<td><p>The ratio of total values to distinct values in a string column to decide whether to prefer dictionary encoding when shuffling the column. If the ratio is higher than this config, dictionary encoding will be used on shuffling string column. This config is effective if it is higher than 1.0. Note that this config is only used when <code class="docutils literal notranslate"><span class="pre">spark.comet.exec.shuffle.mode</span></code> is <code class="docutils literal notranslate"><s [...]
-<td><p>10.0</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.shuffle.sizeInBytesMultiplier</span></code></p></td>
-<td><p>Comet reports smaller sizes for shuffle due to using Arrow’s columnar memory format and this can result in Spark choosing a different join strategy due to the estimated size of the exchange being smaller. Comet will multiple sizeInBytes by this amount to avoid regressions in join strategy.</p></td>
-<td><p>1.0</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-<section id="memory-tuning-configuration-settings">
-<h2>Memory & Tuning Configuration Settings<a class="headerlink" href="#memory-tuning-configuration-settings" title="Link to this heading">#</a></h2>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[tuning]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.batchSize</span></code></p></td>
-<td><p>The columnar batch size, i.e., the maximum number of rows that a batch can contain.</p></td>
-<td><p>8192</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.memoryPool</span></code></p></td>
-<td><p>The type of memory pool to be used for Comet native execution when running Spark in off-heap mode. Available pool types are <code class="docutils literal notranslate"><span class="pre">greedy_unified</span></code> and <code class="docutils literal notranslate"><span class="pre">fair_unified</span></code>. For more information, refer to the <a class="reference external" href="https://datafusion.apache.org/comet/user-guide/tuning.html">Comet Tuning Guide</a>.</p></td>
-<td><p>fair_unified</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.memoryPool.fraction</span></code></p></td>
-<td><p>Fraction of off-heap memory pool that is available to Comet. Only applies to off-heap mode. For more information, refer to the <a class="reference external" href="https://datafusion.apache.org/comet/user-guide/tuning.html">Comet Tuning Guide</a>.</p></td>
-<td><p>1.0</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.tracing.enabled</span></code></p></td>
-<td><p>Enable fine-grained tracing of events and memory usage. For more information, refer to the <a class="reference external" href="https://datafusion.apache.org/comet/user-guide/tracing.html">Comet Tracing Guide</a>.</p></td>
-<td><p>false</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-<section id="development-testing-settings">
-<h2>Development & Testing Settings<a class="headerlink" href="#development-testing-settings" title="Link to this heading">#</a></h2>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[testing]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.memory.factor</span></code></p></td>
-<td><p>Fraction of Comet memory to be allocated per executor process for columnar shuffle when running in on-heap mode. For more information, refer to the <a class="reference external" href="https://datafusion.apache.org/comet/user-guide/tuning.html">Comet Tuning Guide</a>.</p></td>
-<td><p>1.0</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.convert.csv.enabled</span></code></p></td>
-<td><p>When enabled, data from Spark (non-native) CSV v1 and v2 scans will be converted to Arrow format. This is an experimental feature and has known issues with non-UTC timezones.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.convert.json.enabled</span></code></p></td>
-<td><p>When enabled, data from Spark (non-native) JSON v1 and v2 scans will be converted to Arrow format. This is an experimental feature and has known issues with non-UTC timezones.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.convert.parquet.enabled</span></code></p></td>
-<td><p>When enabled, data from Spark (non-native) Parquet v1 and v2 scans will be converted to Arrow format. This is an experimental feature and has known issues with non-UTC timezones.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.onHeap.enabled</span></code></p></td>
-<td><p>Whether to allow Comet to run in on-heap mode. Required for running Spark SQL tests. It can be overridden by the environment variable <code class="docutils literal notranslate"><span class="pre">ENABLE_COMET_ONHEAP</span></code>.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.onHeap.memoryPool</span></code></p></td>
-<td><p>The type of memory pool to be used for Comet native execution when running Spark in on-heap mode. Available pool types are <code class="docutils literal notranslate"><span class="pre">greedy</span></code>, <code class="docutils literal notranslate"><span class="pre">fair_spill</span></code>, <code class="docutils literal notranslate"><span class="pre">greedy_task_shared</span></code>, <code class="docutils literal notranslate"><span class="pre">fair_spill_task_shared</span></code> [...]
-<td><p>greedy_task_shared</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.memoryOverhead</span></code></p></td>
-<td><p>The amount of additional memory to be allocated per executor process for Comet, in MiB, when running Spark in on-heap mode.</p></td>
-<td><p>1024 MiB</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.sparkToColumnar.enabled</span></code></p></td>
-<td><p>Whether to enable Spark to Arrow columnar conversion. When this is turned on, Comet will convert operators in <code class="docutils literal notranslate"><span class="pre">spark.comet.sparkToColumnar.supportedOperatorList</span></code> into Arrow columnar format before processing. This is an experimental feature and has known issues with non-UTC timezones.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.sparkToColumnar.supportedOperatorList</span></code></p></td>
-<td><p>A comma-separated list of operators that will be converted to Arrow columnar format when <code class="docutils literal notranslate"><span class="pre">spark.comet.sparkToColumnar.enabled</span></code> is true.</p></td>
-<td><p>Range,InMemoryTableScan,RDDScan</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.testing.strict</span></code></p></td>
-<td><p>Experimental option to enable strict testing, which will fail tests that could be more comprehensive, such as checking for a specific fallback reason. It can be overridden by the environment variable <code class="docutils literal notranslate"><span class="pre">ENABLE_COMET_STRICT_TESTING</span></code>.</p></td>
-<td><p>false</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-<section id="enabling-or-disabling-individual-operators">
-<h2>Enabling or Disabling Individual Operators<a class="headerlink" href="#enabling-or-disabling-individual-operators" title="Link to this heading">#</a></h2>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[enable_exec]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.aggregate.enabled</span></code></p></td>
-<td><p>Whether to enable aggregate by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.broadcastExchange.enabled</span></code></p></td>
-<td><p>Whether to enable broadcastExchange by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.broadcastHashJoin.enabled</span></code></p></td>
-<td><p>Whether to enable broadcastHashJoin by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.coalesce.enabled</span></code></p></td>
-<td><p>Whether to enable coalesce by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.collectLimit.enabled</span></code></p></td>
-<td><p>Whether to enable collectLimit by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.expand.enabled</span></code></p></td>
-<td><p>Whether to enable expand by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.filter.enabled</span></code></p></td>
-<td><p>Whether to enable filter by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.globalLimit.enabled</span></code></p></td>
-<td><p>Whether to enable globalLimit by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.hashJoin.enabled</span></code></p></td>
-<td><p>Whether to enable hashJoin by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.localLimit.enabled</span></code></p></td>
-<td><p>Whether to enable localLimit by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.localTableScan.enabled</span></code></p></td>
-<td><p>Whether to enable localTableScan by default.</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.project.enabled</span></code></p></td>
-<td><p>Whether to enable project by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.sort.enabled</span></code></p></td>
-<td><p>Whether to enable sort by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.sortMergeJoin.enabled</span></code></p></td>
-<td><p>Whether to enable sortMergeJoin by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.sortMergeJoinWithJoinFilter.enabled</span></code></p></td>
-<td><p>Experimental support for Sort Merge Join with filter</p></td>
-<td><p>false</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.takeOrderedAndProject.enabled</span></code></p></td>
-<td><p>Whether to enable takeOrderedAndProject by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.union.enabled</span></code></p></td>
-<td><p>Whether to enable union by default.</p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.window.enabled</span></code></p></td>
-<td><p>Whether to enable window by default.</p></td>
-<td><p>true</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-<section id="enabling-or-disabling-individual-scalar-expressions">
-<h2>Enabling or Disabling Individual Scalar Expressions<a class="headerlink" href="#enabling-or-disabling-individual-scalar-expressions" title="Link to this heading">#</a></h2>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[enable_expr]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Abs.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Abs</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Acos.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Acos</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Add.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Add</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Alias.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Alias</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.And.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">And</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayAppend.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayAppend</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayCompact.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayCompact</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayContains.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayContains</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayDistinct.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayDistinct</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayExcept.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayExcept</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayFilter.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayFilter</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayInsert.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayInsert</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayIntersect.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayIntersect</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayJoin.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayJoin</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayMax.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayMax</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayMin.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayMin</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayRemove.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayRemove</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayRepeat.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayRepeat</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArrayUnion.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArrayUnion</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ArraysOverlap.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ArraysOverlap</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Ascii.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Ascii</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Asin.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Asin</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Atan.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Atan</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Atan2.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Atan2</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.AttributeReference.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">AttributeReference</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitLength.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitLength</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitwiseAnd.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitwiseAnd</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitwiseCount.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitwiseCount</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitwiseGet.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitwiseGet</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitwiseNot.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitwiseNot</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitwiseOr.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitwiseOr</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitwiseXor.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitwiseXor</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BloomFilterMightContain.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BloomFilterMightContain</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.CaseWhen.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">CaseWhen</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Cast.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Cast</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Ceil.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Ceil</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.CheckOverflow.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">CheckOverflow</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Chr.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Chr</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Coalesce.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Coalesce</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Concat.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Concat</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ConcatWs.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ConcatWs</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Contains.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Contains</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Cos.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Cos</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Cosh.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Cosh</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Cot.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Cot</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.CreateArray.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">CreateArray</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.CreateNamedStruct.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">CreateNamedStruct</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.DateAdd.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">DateAdd</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.DateSub.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">DateSub</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.DayOfMonth.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">DayOfMonth</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.DayOfWeek.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">DayOfWeek</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.DayOfYear.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">DayOfYear</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Divide.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Divide</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ElementAt.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ElementAt</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.EndsWith.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">EndsWith</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.EqualNullSafe.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">EqualNullSafe</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.EqualTo.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">EqualTo</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Exp.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Exp</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Expm1.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Expm1</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Flatten.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Flatten</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Floor.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Floor</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.FromUnixTime.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">FromUnixTime</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.GetArrayItem.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">GetArrayItem</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.GetArrayStructFields.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">GetArrayStructFields</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.GetMapValue.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">GetMapValue</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.GetStructField.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">GetStructField</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.GreaterThan.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">GreaterThan</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.GreaterThanOrEqual.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">GreaterThanOrEqual</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Hex.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Hex</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Hour.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Hour</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.If.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">If</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.In.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">In</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.InSet.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">InSet</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.InitCap.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">InitCap</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.IntegralDivide.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">IntegralDivide</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.IsNaN.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">IsNaN</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.IsNotNull.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">IsNotNull</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.IsNull.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">IsNull</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.KnownFloatingPointNormalized.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">KnownFloatingPointNormalized</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Length.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Length</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.LessThan.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">LessThan</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.LessThanOrEqual.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">LessThanOrEqual</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Like.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Like</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Literal.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Literal</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Log.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Log</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Log10.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Log10</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Log2.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Log2</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Lower.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Lower</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.MakeDecimal.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">MakeDecimal</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.MapEntries.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">MapEntries</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.MapFromArrays.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">MapFromArrays</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.MapKeys.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">MapKeys</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.MapValues.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">MapValues</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Md5.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Md5</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Minute.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Minute</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.MonotonicallyIncreasingID.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">MonotonicallyIncreasingID</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Month.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Month</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Multiply.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Multiply</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Murmur3Hash.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Murmur3Hash</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Not.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Not</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.OctetLength.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">OctetLength</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Or.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Or</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Pow.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Pow</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Quarter.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Quarter</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.RLike.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">RLike</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Rand.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Rand</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Randn.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Randn</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.RegExpReplace.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">RegExpReplace</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Remainder.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Remainder</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Reverse.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Reverse</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Round.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Round</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ScalarSubquery.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ScalarSubquery</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Second.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Second</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Sha1.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Sha1</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Sha2.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Sha2</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ShiftLeft.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ShiftLeft</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.ShiftRight.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">ShiftRight</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Signum.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Signum</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Sin.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Sin</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Sinh.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Sinh</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.SortOrder.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">SortOrder</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.SparkPartitionID.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">SparkPartitionID</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Sqrt.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Sqrt</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StartsWith.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StartsWith</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StaticInvoke.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StaticInvoke</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringInstr.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringInstr</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringLPad.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringLPad</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringRPad.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringRPad</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringRepeat.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringRepeat</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringReplace.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringReplace</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringSpace.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringSpace</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringTranslate.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringTranslate</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringTrim.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringTrim</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringTrimBoth.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringTrimBoth</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringTrimLeft.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringTrimLeft</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StringTrimRight.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StringTrimRight</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StructsToJson.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StructsToJson</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Substring.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Substring</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Subtract.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Subtract</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Tan.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Tan</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Tanh.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Tanh</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.TruncDate.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">TruncDate</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.TruncTimestamp.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">TruncTimestamp</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.UnaryMinus.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">UnaryMinus</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Unhex.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Unhex</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.UnscaledValue.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">UnscaledValue</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Upper.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Upper</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.WeekDay.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">WeekDay</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.WeekOfYear.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">WeekOfYear</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.XxHash64.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">XxHash64</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Year.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Year</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-<section id="enabling-or-disabling-individual-aggregate-expressions">
-<h2>Enabling or Disabling Individual Aggregate Expressions<a class="headerlink" href="#enabling-or-disabling-individual-aggregate-expressions" title="Link to this heading">#</a></h2>
-<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
-<!--BEGIN:CONFIG_TABLE[enable_agg_expr]-->
-<!-- prettier-ignore-start -->
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Default Value</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Average.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Average</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitAndAgg.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitAndAgg</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitOrAgg.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitOrAgg</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BitXorAgg.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BitXorAgg</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.BloomFilterAggregate.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">BloomFilterAggregate</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Corr.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Corr</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Count.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Count</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.CovPopulation.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">CovPopulation</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.CovSample.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">CovSample</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.First.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">First</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Last.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Last</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Max.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Max</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Min.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Min</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StddevPop.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StddevPop</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.StddevSamp.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">StddevSamp</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.Sum.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">Sum</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.VariancePop.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">VariancePop</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.expression.VarianceSamp.enabled</span></code></p></td>
-<td><p>Enable Comet acceleration for <code class="docutils literal notranslate"><span class="pre">VarianceSamp</span></code></p></td>
-<td><p>true</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<!-- prettier-ignore-end -->
-<!--END:CONFIG_TABLE-->
-</section>
-</section>
-
-
- </article>
-
-
-
-
-
- <footer class="prev-next-footer d-print-none">
-
-<div class="prev-next-area">
- <a class="left-prev"
- href="expressions.html"
- title="previous page">
- <i class="fa-solid fa-angle-left"></i>
- <div class="prev-next-info">
- <p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Supported Spark Expressions</p>
- </div>
- </a>
- <a class="right-next"
- href="compatibility.html"
- title="next page">
- <div class="prev-next-info">
- <p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Compatibility Guide</p>
- </div>
- <i class="fa-solid fa-angle-right"></i>
- </a>
-</div>
- </footer>
-
- </div>
-
-
-
-
- </div>
- <footer class="bd-footer-content">
-
- </footer>
-
- </main>
- </div>
- </div>
-
- <!-- Scripts loaded after <body> so the DOM is not blocked -->
- <script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
-<script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
-
-<!-- Based on pydata_sphinx_theme/footer.html -->
-<footer class="footer mt-5 mt-md-0">
- <div class="container">
-
- <div class="footer-item">
- <p>Apache DataFusion, Apache DataFusion Comet, Apache, the Apache feather logo, and the Apache DataFusion project logo</p>
- <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
- </div>
- </div>
-</footer>
-
-
- </body>
-</html>
\ No newline at end of file
diff --git a/user-guide/0.12/datasources.html b/user-guide/0.12/datasources.html
deleted file mode 100644
index 824927d7f..000000000
--- a/user-guide/0.12/datasources.html
+++ /dev/null
@@ -1,730 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-
-<!DOCTYPE html>
-
-
-<html lang="en" data-content_root="../../" data-theme="light">
-
- <head>
- <meta charset="utf-8" />
- <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
-
- <title>Supported Spark Data Sources — Apache DataFusion Comet documentation</title>
-
-
-
- <script data-cfasync="false">
- document.documentElement.dataset.mode = localStorage.getItem("mode") || "light";
- document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
- </script>
- <!--
- this give us a css class that will be invisible only if js is disabled
- -->
- <noscript>
- <style>
- .pst-js-only { display: none !important; }
-
- </style>
- </noscript>
-
- <!-- Loaded before other Sphinx assets -->
- <link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-
- <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=8f2a1f02" />
- <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=cd442bcd" />
-
- <!-- So that users can add custom icons -->
- <script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
- <!-- Pre-loaded scripts that we'll load fully later -->
- <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
-<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
-
- <script src="../../_static/documentation_options.js?v=5929fcd5"></script>
- <script src="../../_static/doctools.js?v=9a2dae69"></script>
- <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
- <script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/0.12/datasources';</script>
- <script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
- <link rel="index" title="Index" href="../../genindex.html" />
- <link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Supported Spark Data Types" href="datatypes.html" />
- <link rel="prev" title="Building Comet From Source" href="source.html" />
- <meta name="viewport" content="width=device-width, initial-scale=1"/>
- <meta name="docsearch:language" content="en"/>
- <meta name="docsearch:version" content="" />
- </head>
-
-
- <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="light">
-
-
-
- <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
-
- <div id="pst-scroll-pixel-helper"></div>
-
- <button type="button" class="btn rounded-pill" id="pst-back-to-top">
- <i class="fa-solid fa-arrow-up"></i>Back to top</button>
-
-
- <dialog id="pst-search-dialog">
-
-<form class="bd-search d-flex align-items-center"
- action="../../search.html"
- method="get">
- <i class="fa-solid fa-magnifying-glass"></i>
- <input type="search"
- class="form-control"
- name="q"
- placeholder="Search the docs ..."
- aria-label="Search the docs ..."
- autocomplete="off"
- autocorrect="off"
- autocapitalize="off"
- spellcheck="false"/>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
-</form>
- </dialog>
-
- <div class="pst-async-banner-revealer d-none">
- <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
-</div>
-
-
- <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
-<div class="bd-header__inner bd-page-width">
- <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
- <span class="fa-solid fa-bars"></span>
- </button>
-
-
- <div class="col-lg-3 navbar-header-items__start">
-
- <div class="navbar-item">
-
-
-
-
-
-<a class="navbar-brand logo" href="../../index.html">
-
-
-
-
-
-
-
-
- <img src="../../_static/DataFusionComet-Logo-Light.png" class="logo__image only-light" alt="Apache DataFusion Comet documentation - Home"/>
- <img src="../../_static/DataFusionComet-Logo-Dark.png" class="logo__image only-dark pst-js-only" alt="Apache DataFusion Comet documentation - Home"/>
-
-
-</a></div>
-
- </div>
-
- <div class="col-lg-9 navbar-header-items">
-
- <div class="me-auto navbar-header-items__center">
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
- </div>
-
-
- <div class="navbar-header-items__end">
-
- <div class="navbar-item navbar-persistent--container">
-
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
-
- <div class="navbar-persistent--mobile">
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
-
-</div>
-
- </header>
-
-
- <div class="bd-container">
- <div class="bd-container__inner bd-page-width">
-
-
-
- <dialog id="pst-primary-sidebar-modal"></dialog>
- <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
-
-
-
- <div class="sidebar-header-items sidebar-primary__section">
-
-
- <div class="sidebar-header-items__center">
-
-
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
-
- </div>
-
-
-
- <div class="sidebar-header-items__end">
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
- <div class="sidebar-primary-items__start sidebar-primary__section">
- <div class="sidebar-primary-item"><!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
- <div class="bd-toc-item active">
- <p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
-<ul class="current">
-<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.12.x</a><ul class="current">
-<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
-<li class="toctree-l3"><a class="reference internal" href="source.html">Building From Source</a></li>
-<li class="toctree-l3 current"><a class="current reference internal" href="#">Supported Data Sources</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datatypes.html">Supported Data Types</a></li>
-<li class="toctree-l3"><a class="reference internal" href="operators.html">Supported Operators</a></li>
-<li class="toctree-l3"><a class="reference internal" href="expressions.html">Supported Expressions</a></li>
-<li class="toctree-l3"><a class="reference internal" href="configs.html">Configuration Settings</a></li>
-<li class="toctree-l3"><a class="reference internal" href="compatibility.html">Compatibility Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="tuning.html">Tuning Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="metrics.html">Metrics Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
-</ul>
-</li>
-</ul>
-</li>
-<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../about/gluten_comparison.html">Comparison with Gluten</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../asf/index.html">ASF Links</a></li>
-</ul>
-
- </div>
-</nav>
-</div>
- </div>
-
-
- <div class="sidebar-primary-items__end sidebar-primary__section">
- <div class="sidebar-primary-item">
-<div id="ethical-ad-placement"
- class="flat"
- data-ea-publisher="readthedocs"
- data-ea-type="readthedocs-sidebar"
- data-ea-manual="true">
-</div></div>
- </div>
-
-
- </div>
-
- <main id="main-content" class="bd-main" role="main">
-
-
- <div class="bd-content">
- <div class="bd-article-container">
-
- <div class="bd-header-article d-print-none">
-<div class="header-article-items header-article__inner">
-
- <div class="header-article-items__start">
-
- <div class="header-article-item">
-
-<nav aria-label="Breadcrumb" class="d-print-none">
- <ul class="bd-breadcrumbs">
-
- <li class="breadcrumb-item breadcrumb-home">
- <a href="../../index.html" class="nav-link" aria-label="Home">
- <i class="fa-solid fa-home"></i>
- </a>
- </li>
-
- <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Comet User Guide</a></li>
-
-
- <li class="breadcrumb-item"><a href="index.html" class="nav-link">Comet 0.12.0 User Guide</a></li>
-
- <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Supported Spark Data Sources</span></li>
- </ul>
-</nav>
-</div>
-
- </div>
-
-
-</div>
-</div>
-
-
-
-
-<div id="searchbox"></div>
- <article class="bd-article">
-
- <!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<div class="admonition warning">
-<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
-</div>
-<section id="supported-spark-data-sources">
-<h1>Supported Spark Data Sources<a class="headerlink" href="#supported-spark-data-sources" title="Link to this heading">#</a></h1>
-<section id="file-formats">
-<h2>File Formats<a class="headerlink" href="#file-formats" title="Link to this heading">#</a></h2>
-<section id="parquet">
-<h3>Parquet<a class="headerlink" href="#parquet" title="Link to this heading">#</a></h3>
-<p>When <code class="docutils literal notranslate"><span class="pre">spark.comet.scan.enabled</span></code> is enabled, Parquet scans will be performed natively by Comet if all data types
-in the schema are supported. When this option is not enabled, the scan will fall back to Spark. In this case,
-enabling <code class="docutils literal notranslate"><span class="pre">spark.comet.convert.parquet.enabled</span></code> will immediately convert the data into Arrow format, allowing native
-execution to happen after that, but the process may not be efficient.</p>
-</section>
-<section id="apache-iceberg">
-<h3>Apache Iceberg<a class="headerlink" href="#apache-iceberg" title="Link to this heading">#</a></h3>
-<p>Comet accelerates Iceberg scans of Parquet files. See the <a class="reference internal" href="iceberg.html"><span class="std std-doc">Iceberg Guide</span></a> for more information.</p>
-</section>
-<section id="csv">
-<h3>CSV<a class="headerlink" href="#csv" title="Link to this heading">#</a></h3>
-<p>Comet does not provide native CSV scan, but when <code class="docutils literal notranslate"><span class="pre">spark.comet.convert.csv.enabled</span></code> is enabled, data is immediately
-converted into Arrow format, allowing native execution to happen after that.</p>
-</section>
-<section id="json">
-<h3>JSON<a class="headerlink" href="#json" title="Link to this heading">#</a></h3>
-<p>Comet does not provide native JSON scan, but when <code class="docutils literal notranslate"><span class="pre">spark.comet.convert.json.enabled</span></code> is enabled, data is immediately
-converted into Arrow format, allowing native execution to happen after that.</p>
-</section>
-</section>
-<section id="data-catalogs">
-<h2>Data Catalogs<a class="headerlink" href="#data-catalogs" title="Link to this heading">#</a></h2>
-<section id="id1">
-<h3>Apache Iceberg<a class="headerlink" href="#id1" title="Link to this heading">#</a></h3>
-<p>See the dedicated <a class="reference internal" href="iceberg.html"><span class="std std-doc">Comet and Iceberg Guide</span></a>.</p>
-</section>
-</section>
-<section id="supported-storages">
-<h2>Supported Storages<a class="headerlink" href="#supported-storages" title="Link to this heading">#</a></h2>
-<p>Comet supports most standard storage systems, such as local file system and object storage.</p>
-<section id="hdfs">
-<h3>HDFS<a class="headerlink" href="#hdfs" title="Link to this heading">#</a></h3>
-<p>Apache DataFusion Comet native reader seamlessly scans files from remote HDFS for <a class="reference internal" href="#supported-spark-data-sources">supported formats</a></p>
-</section>
-<section id="using-experimental-native-datafusion-reader">
-<h3>Using experimental native DataFusion reader<a class="headerlink" href="#using-experimental-native-datafusion-reader" title="Link to this heading">#</a></h3>
-<p>Unlike to native Comet reader the Datafusion reader fully supports nested types processing. This reader is currently experimental only</p>
-<p>To build Comet with native DataFusion reader and remote HDFS support it is required to have a JDK installed</p>
-<p>Example:
-Build a Comet for <code class="docutils literal notranslate"><span class="pre">spark-3.5</span></code> provide a JDK path in <code class="docutils literal notranslate"><span class="pre">JAVA_HOME</span></code>
-Provide the JRE linker path in <code class="docutils literal notranslate"><span class="pre">RUSTFLAGS</span></code>, the path can vary depending on the system. Typically JRE linker is a part of installed JDK</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nb">export</span><span class="w"> </span><span class="nv">JAVA_HOME</span><span class="o">=</span><span class="s2">"/opt/homebrew/opt/openjdk@11"</span>
-make<span class="w"> </span>release<span class="w"> </span><span class="nv">PROFILES</span><span class="o">=</span><span class="s2">"-Pspark-3.5"</span><span class="w"> </span><span class="nv">COMET_FEATURES</span><span class="o">=</span>hdfs<span class="w"> </span><span class="nv">RUSTFLAGS</span><span class="o">=</span><span class="s2">"-L </span><span class="nv">$JAVA_HOME</span><span class="s2">/libexec/openjdk.jdk/Contents/Home/lib/server"</span>
-</pre></div>
-</div>
-<p>Start Comet with experimental reader and HDFS support as <a class="reference internal" href="installation.html#run-spark-shell-with-comet-enabled"><span class="std std-ref">described</span></a>
-and add additional parameters</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>--conf<span class="w"> </span>spark.comet.scan.impl<span class="o">=</span>native_datafusion<span class="w"> </span><span class="se">\</span>
---conf<span class="w"> </span>spark.hadoop.fs.defaultFS<span class="o">=</span><span class="s2">"hdfs://namenode:9000"</span><span class="w"> </span><span class="se">\</span>
---conf<span class="w"> </span>spark.hadoop.dfs.client.use.datanode.hostname<span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
---conf<span class="w"> </span>dfs.client.use.datanode.hostname<span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">true</span>
-</pre></div>
-</div>
-<p>Query a struct type from Remote HDFS</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>spark.read.parquet<span class="o">(</span><span class="s2">"hdfs://namenode:9000/user/data"</span><span class="o">)</span>.show<span class="o">(</span><span class="nb">false</span><span class="o">)</span>
-
-root
-<span class="w"> </span><span class="p">|</span>--<span class="w"> </span>id:<span class="w"> </span>integer<span class="w"> </span><span class="o">(</span><span class="nv">nullable</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">true</span><span class="o">)</span>
-<span class="w"> </span><span class="p">|</span>--<span class="w"> </span>first_name:<span class="w"> </span>string<span class="w"> </span><span class="o">(</span><span class="nv">nullable</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">true</span><span class="o">)</span>
-<span class="w"> </span><span class="p">|</span>--<span class="w"> </span>personal_info:<span class="w"> </span>struct<span class="w"> </span><span class="o">(</span><span class="nv">nullable</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">true</span><span class="o">)</span>
-<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="p">|</span>--<span class="w"> </span>firstName:<span class="w"> </span>string<span class="w"> </span><span class="o">(</span><span class="nv">nullable</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">true</span><span class="o">)</span>
-<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="p">|</span>--<span class="w"> </span>lastName:<span class="w"> </span>string<span class="w"> </span><span class="o">(</span><span class="nv">nullable</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">true</span><span class="o">)</span>
-<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="p">|</span>--<span class="w"> </span>ageInYears:<span class="w"> </span>integer<span class="w"> </span><span class="o">(</span><span class="nv">nullable</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">true</span><span class="o">)</span>
-
-<span class="m">25</span>/01/30<span class="w"> </span><span class="m">16</span>:50:43<span class="w"> </span>INFO<span class="w"> </span>core/src/lib.rs:<span class="w"> </span>Comet<span class="w"> </span>native<span class="w"> </span>library<span class="w"> </span>version<span class="w"> </span><span class="m">0</span>.12.0<span class="w"> </span><span class="nv">initialized</span>
-<span class="o">==</span><span class="w"> </span>Physical<span class="w"> </span><span class="nv">Plan</span><span class="w"> </span><span class="o">==</span>
-*<span class="w"> </span>CometColumnarToRow<span class="w"> </span><span class="o">(</span><span class="m">2</span><span class="o">)</span>
-+-<span class="w"> </span>CometNativeScan:<span class="w"> </span><span class="o">(</span><span class="m">1</span><span class="o">)</span>
-
-
-<span class="o">(</span><span class="m">1</span><span class="o">)</span><span class="w"> </span>CometNativeScan:
-Output<span class="w"> </span><span class="o">[</span><span class="m">3</span><span class="o">]</span>:<span class="w"> </span><span class="o">[</span>id#0,<span class="w"> </span>first_name#1,<span class="w"> </span>personal_info#4<span class="o">]</span>
-Arguments:<span class="w"> </span><span class="o">[</span>id#0,<span class="w"> </span>first_name#1,<span class="w"> </span>personal_info#4<span class="o">]</span>
-
-<span class="o">(</span><span class="m">2</span><span class="o">)</span><span class="w"> </span>CometColumnarToRow<span class="w"> </span><span class="o">[</span>codegen<span class="w"> </span>id<span class="w"> </span>:<span class="w"> </span><span class="m">1</span><span class="o">]</span>
-Input<span class="w"> </span><span class="o">[</span><span class="m">3</span><span class="o">]</span>:<span class="w"> </span><span class="o">[</span>id#0,<span class="w"> </span>first_name#1,<span class="w"> </span>personal_info#4<span class="o">]</span>
-
-
-<span class="m">25</span>/01/30<span class="w"> </span><span class="m">16</span>:50:44<span class="w"> </span>INFO<span class="w"> </span>fs-hdfs-0.1.12/src/hdfs.rs:<span class="w"> </span>Connecting<span class="w"> </span>to<span class="w"> </span>Namenode<span class="w"> </span><span class="o">(</span>hdfs://namenode:9000<span class="o">)</span>
-+---+----------+-----------------+
-<span class="p">|</span>id<span class="w"> </span><span class="p">|</span>first_name<span class="p">|</span>personal_info<span class="w"> </span><span class="p">|</span>
-+---+----------+-----------------+
-<span class="p">|</span><span class="m">2</span><span class="w"> </span><span class="p">|</span>Jane<span class="w"> </span><span class="p">|</span><span class="o">{</span>Jane,<span class="w"> </span>Smith,<span class="w"> </span><span class="m">34</span><span class="o">}</span><span class="p">|</span>
-<span class="p">|</span><span class="m">1</span><span class="w"> </span><span class="p">|</span>John<span class="w"> </span><span class="p">|</span><span class="o">{</span>John,<span class="w"> </span>Doe,<span class="w"> </span><span class="m">28</span><span class="o">}</span><span class="w"> </span><span class="p">|</span>
-+---+----------+-----------------+
-</pre></div>
-</div>
-<p>Verify the native scan type should be <code class="docutils literal notranslate"><span class="pre">CometNativeScan</span></code>.</p>
-<p>More on <a class="reference external" href="https://github.com/apache/datafusion-comet/blob/main/native/hdfs/README.md">HDFS Reader</a></p>
-</section>
-<section id="local-hdfs-development">
-<h3>Local HDFS development<a class="headerlink" href="#local-hdfs-development" title="Link to this heading">#</a></h3>
-<ul class="simple">
-<li><p>Configure local machine network. Add hostname to <code class="docutils literal notranslate"><span class="pre">/etc/hosts</span></code></p></li>
-</ul>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="m">127</span>.0.0.1<span class="w"> </span>localhost<span class="w"> </span>namenode<span class="w"> </span>datanode1<span class="w"> </span>datanode2<span class="w"> </span>datanode3
-::1<span class="w"> </span>localhost<span class="w"> </span>namenode<span class="w"> </span>datanode1<span class="w"> </span>datanode2<span class="w"> </span>datanode3
-</pre></div>
-</div>
-<ul class="simple">
-<li><p>Start local HDFS cluster, 3 datanodes, namenode url is <code class="docutils literal notranslate"><span class="pre">namenode:9000</span></code></p></li>
-</ul>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>docker<span class="w"> </span>compose<span class="w"> </span>-f<span class="w"> </span>kube/local/hdfs-docker-compose.yml<span class="w"> </span>up
-</pre></div>
-</div>
-<ul class="simple">
-<li><p>Check the local namenode is up and running on <code class="docutils literal notranslate"><span class="pre">http://localhost:9870/dfshealth.html#tab-overview</span></code></p></li>
-<li><p>Build a project with HDFS support</p></li>
-</ul>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nv">JAVA_HOME</span><span class="o">=</span><span class="s2">"/opt/homebrew/opt/openjdk@11"</span><span class="w"> </span>make<span class="w"> </span>release<span class="w"> </span><span class="nv">PROFILES</span><span class="o">=</span><span class="s2">"-Pspark-3.5"</span><span class="w"> </span><span class="nv">COMET_FEATURES</span><span class="o">=</span>hdfs<span class=" [...]
-</pre></div>
-</div>
-<ul class="simple">
-<li><p>Run local test</p></li>
-</ul>
-<div class="highlight-scala notranslate"><div class="highlight"><pre><span></span><span class="w"> </span><span class="n">withSQLConf</span><span class="p">(</span>
-<span class="w"> </span><span class="nc">CometConf</span><span class="p">.</span><span class="nc">COMET_ENABLED</span><span class="p">.</span><span class="n">key</span><span class="w"> </span><span class="o">-></span><span class="w"> </span><span class="s">"true"</span><span class="p">,</span>
-<span class="w"> </span><span class="nc">CometConf</span><span class="p">.</span><span class="nc">COMET_EXEC_ENABLED</span><span class="p">.</span><span class="n">key</span><span class="w"> </span><span class="o">-></span><span class="w"> </span><span class="s">"true"</span><span class="p">,</span>
-<span class="w"> </span><span class="nc">CometConf</span><span class="p">.</span><span class="nc">COMET_NATIVE_SCAN_IMPL</span><span class="p">.</span><span class="n">key</span><span class="w"> </span><span class="o">-></span><span class="w"> </span><span class="nc">CometConf</span><span class="p">.</span><span class="nc">SCAN_NATIVE_DATAFUSION</span><span class="p">,</span>
-<span class="w"> </span><span class="nc">SQLConf</span><span class="p">.</span><span class="nc">USE_V1_SOURCE_LIST</span><span class="p">.</span><span class="n">key</span><span class="w"> </span><span class="o">-></span><span class="w"> </span><span class="s">"parquet"</span><span class="p">,</span>
-<span class="w"> </span><span class="s">"fs.defaultFS"</span><span class="w"> </span><span class="o">-></span><span class="w"> </span><span class="s">"hdfs://namenode:9000"</span><span class="p">,</span>
-<span class="w"> </span><span class="s">"dfs.client.use.datanode.hostname"</span><span class="w"> </span><span class="o">-></span><span class="w"> </span><span class="s">"true"</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
-<span class="w"> </span><span class="kd">val</span><span class="w"> </span><span class="n">df</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">spark</span><span class="p">.</span><span class="n">read</span><span class="p">.</span><span class="n">parquet</span><span class="p">(</span><span class="s">"/tmp/2"</span><span class="p">)</span>
-<span class="w"> </span><span class="n">df</span><span class="p">.</span><span class="n">show</span><span class="p">(</span><span class="kc">false</span><span class="p">)</span>
-<span class="w"> </span><span class="n">df</span><span class="p">.</span><span class="n">explain</span><span class="p">(</span><span class="s">"extended"</span><span class="p">)</span>
-<span class="w"> </span><span class="p">}</span>
-<span class="w"> </span><span class="p">}</span>
-</pre></div>
-</div>
-<p>Or use <code class="docutils literal notranslate"><span class="pre">spark-shell</span></code> with HDFS support as described <a class="reference internal" href="#using-experimental-native-datafusion-reader">above</a></p>
-</section>
-</section>
-<section id="s3">
-<h2>S3<a class="headerlink" href="#s3" title="Link to this heading">#</a></h2>
-<section id="root-ca-certificates">
-<h3>Root CA Certificates<a class="headerlink" href="#root-ca-certificates" title="Link to this heading">#</a></h3>
-<p>One major difference between Spark and Comet is the mechanism for discovering Root
-CA Certificates. Spark uses the JVM to read CA Certificates from the Java Trust Store, but native Comet
-scans use system Root CA Certificates (typically stored
-in <code class="docutils literal notranslate"><span class="pre">/etc/ssl/certs</span></code> on Linux). These scans will not be able to interact with S3 if the Root CA Certificates are not
-installed.</p>
-</section>
-<section id="supported-credential-providers">
-<h3>Supported Credential Providers<a class="headerlink" href="#supported-credential-providers" title="Link to this heading">#</a></h3>
-<p>AWS credential providers can be configured using the <code class="docutils literal notranslate"><span class="pre">fs.s3a.aws.credentials.provider</span></code> configuration. The following table shows the supported credential providers and their configuration options:</p>
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Credential provider</p></th>
-<th class="head"><p>Description</p></th>
-<th class="head"><p>Supported Options</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</span></code></p></td>
-<td><p>Access S3 using access key and secret key</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">fs.s3a.access.key</span></code>, <code class="docutils literal notranslate"><span class="pre">fs.s3a.secret.key</span></code></p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider</span></code></p></td>
-<td><p>Access S3 using temporary credentials</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">fs.s3a.access.key</span></code>, <code class="docutils literal notranslate"><span class="pre">fs.s3a.secret.key</span></code>, <code class="docutils literal notranslate"><span class="pre">fs.s3a.session.token</span></code></p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider</span></code></p></td>
-<td><p>Access S3 using AWS STS assume role</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">fs.s3a.assumed.role.arn</span></code>, <code class="docutils literal notranslate"><span class="pre">fs.s3a.assumed.role.session.name</span></code> (optional), <code class="docutils literal notranslate"><span class="pre">fs.s3a.assumed.role.credentials.provider</span></code> (optional)</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider</span></code></p></td>
-<td><p>Access S3 using EC2 instance profile or ECS task credentials (tries ECS first, then IMDS)</p></td>
-<td><p>None (auto-detected)</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider</span></code><br/><code class="docutils literal notranslate"><span class="pre">com.amazonaws.auth.AnonymousAWSCredentials</span></code><br/><code class="docutils literal notranslate"><span class="pre">software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider</span></code></p></td>
-<td><p>Access S3 without authentication (public buckets only)</p></td>
-<td><p>None</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">com.amazonaws.auth.EnvironmentVariableCredentialsProvider</span></code><br/><code class="docutils literal notranslate"><span class="pre">software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider</span></code></p></td>
-<td><p>Load credentials from environment variables (<code class="docutils literal notranslate"><span class="pre">AWS_ACCESS_KEY_ID</span></code>, <code class="docutils literal notranslate"><span class="pre">AWS_SECRET_ACCESS_KEY</span></code>, <code class="docutils literal notranslate"><span class="pre">AWS_SESSION_TOKEN</span></code>)</p></td>
-<td><p>None</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">com.amazonaws.auth.InstanceProfileCredentialsProvider</span></code><br/><code class="docutils literal notranslate"><span class="pre">software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider</span></code></p></td>
-<td><p>Access S3 using EC2 instance metadata service (IMDS)</p></td>
-<td><p>None</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">com.amazonaws.auth.ContainerCredentialsProvider</span></code><br/><code class="docutils literal notranslate"><span class="pre">software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider</span></code><br/><code class="docutils literal notranslate"><span class="pre">com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper</span></code></p></td>
-<td><p>Access S3 using ECS task credentials</p></td>
-<td><p>None</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">com.amazonaws.auth.WebIdentityTokenCredentialsProvider</span></code><br/><code class="docutils literal notranslate"><span class="pre">software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider</span></code></p></td>
-<td><p>Authenticate using web identity token file</p></td>
-<td><p>None</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<p>Multiple credential providers can be specified in a comma-separated list using the <code class="docutils literal notranslate"><span class="pre">fs.s3a.aws.credentials.provider</span></code> configuration, just as Hadoop AWS supports. If <code class="docutils literal notranslate"><span class="pre">fs.s3a.aws.credentials.provider</span></code> is not configured, Hadoop S3A’s default credential provider chain will be used. All configuration options also support bucket-specific overrides [...]
-</section>
-</section>
-</section>
-
-
- </article>
-
-
-
-
-
- <footer class="prev-next-footer d-print-none">
-
-<div class="prev-next-area">
- <a class="left-prev"
- href="source.html"
- title="previous page">
- <i class="fa-solid fa-angle-left"></i>
- <div class="prev-next-info">
- <p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Building Comet From Source</p>
- </div>
- </a>
- <a class="right-next"
- href="datatypes.html"
- title="next page">
- <div class="prev-next-info">
- <p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Supported Spark Data Types</p>
- </div>
- <i class="fa-solid fa-angle-right"></i>
- </a>
-</div>
- </footer>
-
- </div>
-
-
-
-
- </div>
- <footer class="bd-footer-content">
-
- </footer>
-
- </main>
- </div>
- </div>
-
- <!-- Scripts loaded after <body> so the DOM is not blocked -->
- <script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
-<script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
-
-<!-- Based on pydata_sphinx_theme/footer.html -->
-<footer class="footer mt-5 mt-md-0">
- <div class="container">
-
- <div class="footer-item">
- <p>Apache DataFusion, Apache DataFusion Comet, Apache, the Apache feather logo, and the Apache DataFusion project logo</p>
- <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
- </div>
- </div>
-</footer>
-
-
- </body>
-</html>
\ No newline at end of file
diff --git a/user-guide/0.12/iceberg.html b/user-guide/0.12/iceberg.html
deleted file mode 100644
index bc7f006c0..000000000
--- a/user-guide/0.12/iceberg.html
+++ /dev/null
@@ -1,644 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-
-<!DOCTYPE html>
-
-
-<html lang="en" data-content_root="../../" data-theme="light">
-
- <head>
- <meta charset="utf-8" />
- <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
-
- <title>Accelerating Apache Iceberg Parquet Scans using Comet (Experimental) — Apache DataFusion Comet documentation</title>
-
-
-
- <script data-cfasync="false">
- document.documentElement.dataset.mode = localStorage.getItem("mode") || "light";
- document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
- </script>
- <!--
- this give us a css class that will be invisible only if js is disabled
- -->
- <noscript>
- <style>
- .pst-js-only { display: none !important; }
-
- </style>
- </noscript>
-
- <!-- Loaded before other Sphinx assets -->
- <link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-
- <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=8f2a1f02" />
- <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=cd442bcd" />
-
- <!-- So that users can add custom icons -->
- <script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
- <!-- Pre-loaded scripts that we'll load fully later -->
- <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
-<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
-
- <script src="../../_static/documentation_options.js?v=5929fcd5"></script>
- <script src="../../_static/doctools.js?v=9a2dae69"></script>
- <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
- <script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/0.12/iceberg';</script>
- <script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
- <link rel="index" title="Index" href="../../genindex.html" />
- <link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Comet Kubernetes Support" href="kubernetes.html" />
- <link rel="prev" title="Comet Metrics" href="metrics.html" />
- <meta name="viewport" content="width=device-width, initial-scale=1"/>
- <meta name="docsearch:language" content="en"/>
- <meta name="docsearch:version" content="" />
- </head>
-
-
- <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="light">
-
-
-
- <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
-
- <div id="pst-scroll-pixel-helper"></div>
-
- <button type="button" class="btn rounded-pill" id="pst-back-to-top">
- <i class="fa-solid fa-arrow-up"></i>Back to top</button>
-
-
- <dialog id="pst-search-dialog">
-
-<form class="bd-search d-flex align-items-center"
- action="../../search.html"
- method="get">
- <i class="fa-solid fa-magnifying-glass"></i>
- <input type="search"
- class="form-control"
- name="q"
- placeholder="Search the docs ..."
- aria-label="Search the docs ..."
- autocomplete="off"
- autocorrect="off"
- autocapitalize="off"
- spellcheck="false"/>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
-</form>
- </dialog>
-
- <div class="pst-async-banner-revealer d-none">
- <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
-</div>
-
-
- <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
-<div class="bd-header__inner bd-page-width">
- <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
- <span class="fa-solid fa-bars"></span>
- </button>
-
-
- <div class="col-lg-3 navbar-header-items__start">
-
- <div class="navbar-item">
-
-
-
-
-
-<a class="navbar-brand logo" href="../../index.html">
-
-
-
-
-
-
-
-
- <img src="../../_static/DataFusionComet-Logo-Light.png" class="logo__image only-light" alt="Apache DataFusion Comet documentation - Home"/>
- <img src="../../_static/DataFusionComet-Logo-Dark.png" class="logo__image only-dark pst-js-only" alt="Apache DataFusion Comet documentation - Home"/>
-
-
-</a></div>
-
- </div>
-
- <div class="col-lg-9 navbar-header-items">
-
- <div class="me-auto navbar-header-items__center">
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
- </div>
-
-
- <div class="navbar-header-items__end">
-
- <div class="navbar-item navbar-persistent--container">
-
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
-
- <div class="navbar-persistent--mobile">
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
-
-</div>
-
- </header>
-
-
- <div class="bd-container">
- <div class="bd-container__inner bd-page-width">
-
-
-
- <dialog id="pst-primary-sidebar-modal"></dialog>
- <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
-
-
-
- <div class="sidebar-header-items sidebar-primary__section">
-
-
- <div class="sidebar-header-items__center">
-
-
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
-
- </div>
-
-
-
- <div class="sidebar-header-items__end">
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
- <div class="sidebar-primary-items__start sidebar-primary__section">
- <div class="sidebar-primary-item"><!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
- <div class="bd-toc-item active">
- <p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
-<ul class="current">
-<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.12.x</a><ul class="current">
-<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
-<li class="toctree-l3"><a class="reference internal" href="source.html">Building From Source</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datasources.html">Supported Data Sources</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datatypes.html">Supported Data Types</a></li>
-<li class="toctree-l3"><a class="reference internal" href="operators.html">Supported Operators</a></li>
-<li class="toctree-l3"><a class="reference internal" href="expressions.html">Supported Expressions</a></li>
-<li class="toctree-l3"><a class="reference internal" href="configs.html">Configuration Settings</a></li>
-<li class="toctree-l3"><a class="reference internal" href="compatibility.html">Compatibility Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="tuning.html">Tuning Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="metrics.html">Metrics Guide</a></li>
-<li class="toctree-l3 current"><a class="current reference internal" href="#">Iceberg Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
-</ul>
-</li>
-</ul>
-</li>
-<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../about/gluten_comparison.html">Comparison with Gluten</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../asf/index.html">ASF Links</a></li>
-</ul>
-
- </div>
-</nav>
-</div>
- </div>
-
-
- <div class="sidebar-primary-items__end sidebar-primary__section">
- <div class="sidebar-primary-item">
-<div id="ethical-ad-placement"
- class="flat"
- data-ea-publisher="readthedocs"
- data-ea-type="readthedocs-sidebar"
- data-ea-manual="true">
-</div></div>
- </div>
-
-
- </div>
-
- <main id="main-content" class="bd-main" role="main">
-
-
- <div class="bd-content">
- <div class="bd-article-container">
-
- <div class="bd-header-article d-print-none">
-<div class="header-article-items header-article__inner">
-
- <div class="header-article-items__start">
-
- <div class="header-article-item">
-
-<nav aria-label="Breadcrumb" class="d-print-none">
- <ul class="bd-breadcrumbs">
-
- <li class="breadcrumb-item breadcrumb-home">
- <a href="../../index.html" class="nav-link" aria-label="Home">
- <i class="fa-solid fa-home"></i>
- </a>
- </li>
-
- <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Comet User Guide</a></li>
-
-
- <li class="breadcrumb-item"><a href="index.html" class="nav-link">Comet 0.12.0 User Guide</a></li>
-
- <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Accelerating Apache Iceberg Parquet Scans using Comet (Experimental)</span></li>
- </ul>
-</nav>
-</div>
-
- </div>
-
-
-</div>
-</div>
-
-
-
-
-<div id="searchbox"></div>
- <article class="bd-article">
-
- <!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<div class="admonition warning">
-<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
-</div>
-<section id="accelerating-apache-iceberg-parquet-scans-using-comet-experimental">
-<h1>Accelerating Apache Iceberg Parquet Scans using Comet (Experimental)<a class="headerlink" href="#accelerating-apache-iceberg-parquet-scans-using-comet-experimental" title="Link to this heading">#</a></h1>
-<p><strong>Note: Iceberg integration is a work-in-progress. It is currently necessary to build Iceberg from
-source rather than using available artifacts in Maven</strong></p>
-<section id="build-comet">
-<h2>Build Comet<a class="headerlink" href="#build-comet" title="Link to this heading">#</a></h2>
-<p>Run a Maven install so that we can compile Iceberg against latest Comet:</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>mvn<span class="w"> </span>install<span class="w"> </span>-DskipTests
-</pre></div>
-</div>
-<p>Build the release JAR to be used from Spark:</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>make<span class="w"> </span>release
-</pre></div>
-</div>
-<p>Set <code class="docutils literal notranslate"><span class="pre">COMET_JAR</span></code> env var:</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nb">export</span><span class="w"> </span><span class="nv">COMET_JAR</span><span class="o">=</span><span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span>/spark/target/comet-spark-spark3.5_2.12-0.12.0.jar
-</pre></div>
-</div>
-</section>
-<section id="build-iceberg">
-<h2>Build Iceberg<a class="headerlink" href="#build-iceberg" title="Link to this heading">#</a></h2>
-<p>Clone the Iceberg repository and apply code changes needed by Comet</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>git<span class="w"> </span>clone<span class="w"> </span>git@github.com:apache/iceberg.git
-<span class="nb">cd</span><span class="w"> </span>iceberg
-git<span class="w"> </span>checkout<span class="w"> </span>apache-iceberg-1.8.1
-git<span class="w"> </span>apply<span class="w"> </span>../datafusion-comet/dev/diffs/iceberg/1.8.1.diff
-</pre></div>
-</div>
-<p>Perform a clean build</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>./gradlew<span class="w"> </span>clean
-./gradlew<span class="w"> </span>build<span class="w"> </span>-x<span class="w"> </span><span class="nb">test</span><span class="w"> </span>-x<span class="w"> </span>integrationTest
-</pre></div>
-</div>
-</section>
-<section id="test">
-<h2>Test<a class="headerlink" href="#test" title="Link to this heading">#</a></h2>
-<p>Set <code class="docutils literal notranslate"><span class="pre">ICEBERG_JAR</span></code> environment variable.</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nb">export</span><span class="w"> </span><span class="nv">ICEBERG_JAR</span><span class="o">=</span><span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span>/spark/v3.5/spark-runtime/build/libs/iceberg-spark-runtime-3.5_2.12-1.9.0-SNAPSHOT.jar
-</pre></div>
-</div>
-<p>Launch Spark Shell:</p>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nv">$SPARK_HOME</span>/bin/spark-shell<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--jars<span class="w"> </span><span class="nv">$COMET_JAR</span>,<span class="nv">$ICEBERG_JAR</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.driver.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span>,<span class="nv">$ICEBERG_JAR</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span>,<span class="nv">$ICEBERG_JAR</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.sql.extensions<span class="o">=</span>org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.sql.catalog.spark_catalog<span class="o">=</span>org.apache.iceberg.spark.SparkCatalog<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.sql.catalog.spark_catalog.type<span class="o">=</span>hadoop<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.sql.catalog.spark_catalog.warehouse<span class="o">=</span>/tmp/warehouse<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.plugins<span class="o">=</span>org.apache.spark.CometPlugin<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.shuffle.manager<span class="o">=</span>org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.sql.iceberg.parquet.reader-type<span class="o">=</span>COMET<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.explainFallback.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.size<span class="o">=</span>2g<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.use.lazyMaterialization<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.schemaEvolution.enabled<span class="o">=</span><span class="nb">true</span>
-</pre></div>
-</div>
-<p>Create an Iceberg table. Note that Comet will not accelerate this part.</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">scala</span><span class="o">></span> <span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">s</span><span class="s2">"CREATE TABLE IF NOT EXISTS t1 (c0 INT, c1 STRING) USING iceberg"</span><span class="p">)</span>
-<span class="n">scala</span><span class="o">></span> <span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">s</span><span class="s2">"INSERT INTO t1 VALUES ${(0 until 10000).map(i => (i, i)).mkString("</span><span class="p">,</span><span class="s2">")}"</span><span class="p">)</span>
-</pre></div>
-</div>
-<p>Comet should now be able to accelerate reading the table:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">scala</span><span class="o">></span> <span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">s</span><span class="s2">"SELECT * from t1"</span><span class="p">)</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
-</pre></div>
-</div>
-<p>This should produce the following output:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">scala</span><span class="o">></span> <span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">s</span><span class="s2">"SELECT * from t1"</span><span class="p">)</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
-<span class="o">+---+---+</span>
-<span class="o">|</span> <span class="n">c0</span><span class="o">|</span> <span class="n">c1</span><span class="o">|</span>
-<span class="o">+---+---+</span>
-<span class="o">|</span> <span class="mi">0</span><span class="o">|</span> <span class="mi">0</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">1</span><span class="o">|</span> <span class="mi">1</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">2</span><span class="o">|</span> <span class="mi">2</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">3</span><span class="o">|</span> <span class="mi">3</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">4</span><span class="o">|</span> <span class="mi">4</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">5</span><span class="o">|</span> <span class="mi">5</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">6</span><span class="o">|</span> <span class="mi">6</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">7</span><span class="o">|</span> <span class="mi">7</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">8</span><span class="o">|</span> <span class="mi">8</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">9</span><span class="o">|</span> <span class="mi">9</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">10</span><span class="o">|</span> <span class="mi">10</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">11</span><span class="o">|</span> <span class="mi">11</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">12</span><span class="o">|</span> <span class="mi">12</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">13</span><span class="o">|</span> <span class="mi">13</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">14</span><span class="o">|</span> <span class="mi">14</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">15</span><span class="o">|</span> <span class="mi">15</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">16</span><span class="o">|</span> <span class="mi">16</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">17</span><span class="o">|</span> <span class="mi">17</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">18</span><span class="o">|</span> <span class="mi">18</span><span class="o">|</span>
-<span class="o">|</span> <span class="mi">19</span><span class="o">|</span> <span class="mi">19</span><span class="o">|</span>
-<span class="o">+---+---+</span>
-<span class="n">only</span> <span class="n">showing</span> <span class="n">top</span> <span class="mi">20</span> <span class="n">rows</span>
-</pre></div>
-</div>
-<p>Confirm that the query was accelerated by Comet:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">scala</span><span class="o">></span> <span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">s</span><span class="s2">"SELECT * from t1"</span><span class="p">)</span><span class="o">.</span><span class="n">explain</span><span class="p">()</span>
-<span class="o">==</span> <span class="n">Physical</span> <span class="n">Plan</span> <span class="o">==</span>
-<span class="o">*</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="n">CometColumnarToRow</span>
-<span class="o">+-</span> <span class="n">CometBatchScan</span> <span class="n">spark_catalog</span><span class="o">.</span><span class="n">default</span><span class="o">.</span><span class="n">t1</span><span class="p">[</span><span class="n">c0</span><span class="c1">#26, c1#27] spark_catalog.default.t1 (branch=null) [filters=, groupedBy=] RuntimeFilters: []</span>
-</pre></div>
-</div>
-</section>
-<section id="known-issues">
-<h2>Known issues<a class="headerlink" href="#known-issues" title="Link to this heading">#</a></h2>
-<ul class="simple">
-<li><p>Spark Runtime Filtering isn’t <a class="reference external" href="https://github.com/apache/datafusion-comet/issues/2116">working</a></p>
-<ul>
-<li><p>You can bypass the issue by either setting <code class="docutils literal notranslate"><span class="pre">spark.sql.adaptive.enabled=false</span></code> or <code class="docutils literal notranslate"><span class="pre">spark.comet.exec.broadcastExchange.enabled=false</span></code></p></li>
-</ul>
-</li>
-</ul>
-</section>
-</section>
-
-
- </article>
-
-
-
-
-
- <footer class="prev-next-footer d-print-none">
-
-<div class="prev-next-area">
- <a class="left-prev"
- href="metrics.html"
- title="previous page">
- <i class="fa-solid fa-angle-left"></i>
- <div class="prev-next-info">
- <p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Comet Metrics</p>
- </div>
- </a>
- <a class="right-next"
- href="kubernetes.html"
- title="next page">
- <div class="prev-next-info">
- <p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Comet Kubernetes Support</p>
- </div>
- <i class="fa-solid fa-angle-right"></i>
- </a>
-</div>
- </footer>
-
- </div>
-
-
-
-
- </div>
- <footer class="bd-footer-content">
-
- </footer>
-
- </main>
- </div>
- </div>
-
- <!-- Scripts loaded after <body> so the DOM is not blocked -->
- <script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
-<script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
-
-<!-- Based on pydata_sphinx_theme/footer.html -->
-<footer class="footer mt-5 mt-md-0">
- <div class="container">
-
- <div class="footer-item">
- <p>Apache DataFusion, Apache DataFusion Comet, Apache, the Apache feather logo, and the Apache DataFusion project logo</p>
- <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
- </div>
- </div>
-</footer>
-
-
- </body>
-</html>
\ No newline at end of file
diff --git a/user-guide/0.12/operators.html b/user-guide/0.12/operators.html
deleted file mode 100644
index 3ee8bd8a5..000000000
--- a/user-guide/0.12/operators.html
+++ /dev/null
@@ -1,616 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-
-<!DOCTYPE html>
-
-
-<html lang="en" data-content_root="../../" data-theme="light">
-
- <head>
- <meta charset="utf-8" />
- <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
-
- <title>Supported Spark Operators — Apache DataFusion Comet documentation</title>
-
-
-
- <script data-cfasync="false">
- document.documentElement.dataset.mode = localStorage.getItem("mode") || "light";
- document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
- </script>
- <!--
- this give us a css class that will be invisible only if js is disabled
- -->
- <noscript>
- <style>
- .pst-js-only { display: none !important; }
-
- </style>
- </noscript>
-
- <!-- Loaded before other Sphinx assets -->
- <link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
-
- <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=8f2a1f02" />
- <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=cd442bcd" />
-
- <!-- So that users can add custom icons -->
- <script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
- <!-- Pre-loaded scripts that we'll load fully later -->
- <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
-<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
-
- <script src="../../_static/documentation_options.js?v=5929fcd5"></script>
- <script src="../../_static/doctools.js?v=9a2dae69"></script>
- <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
- <script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/0.12/operators';</script>
- <script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
- <link rel="index" title="Index" href="../../genindex.html" />
- <link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Supported Spark Expressions" href="expressions.html" />
- <link rel="prev" title="Supported Spark Data Types" href="datatypes.html" />
- <meta name="viewport" content="width=device-width, initial-scale=1"/>
- <meta name="docsearch:language" content="en"/>
- <meta name="docsearch:version" content="" />
- </head>
-
-
- <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="light">
-
-
-
- <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
-
- <div id="pst-scroll-pixel-helper"></div>
-
- <button type="button" class="btn rounded-pill" id="pst-back-to-top">
- <i class="fa-solid fa-arrow-up"></i>Back to top</button>
-
-
- <dialog id="pst-search-dialog">
-
-<form class="bd-search d-flex align-items-center"
- action="../../search.html"
- method="get">
- <i class="fa-solid fa-magnifying-glass"></i>
- <input type="search"
- class="form-control"
- name="q"
- placeholder="Search the docs ..."
- aria-label="Search the docs ..."
- autocomplete="off"
- autocorrect="off"
- autocapitalize="off"
- spellcheck="false"/>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
-</form>
- </dialog>
-
- <div class="pst-async-banner-revealer d-none">
- <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
-</div>
-
-
- <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
-<div class="bd-header__inner bd-page-width">
- <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
- <span class="fa-solid fa-bars"></span>
- </button>
-
-
- <div class="col-lg-3 navbar-header-items__start">
-
- <div class="navbar-item">
-
-
-
-
-
-<a class="navbar-brand logo" href="../../index.html">
-
-
-
-
-
-
-
-
- <img src="../../_static/DataFusionComet-Logo-Light.png" class="logo__image only-light" alt="Apache DataFusion Comet documentation - Home"/>
- <img src="../../_static/DataFusionComet-Logo-Dark.png" class="logo__image only-dark pst-js-only" alt="Apache DataFusion Comet documentation - Home"/>
-
-
-</a></div>
-
- </div>
-
- <div class="col-lg-9 navbar-header-items">
-
- <div class="me-auto navbar-header-items__center">
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
- </div>
-
-
- <div class="navbar-header-items__end">
-
- <div class="navbar-item navbar-persistent--container">
-
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
-
- <div class="navbar-persistent--mobile">
-
-<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="fa-solid fa-magnifying-glass"></i>
- <span class="search-button__default-text">Search</span>
- <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
-</button>
- </div>
-
-
-
-</div>
-
- </header>
-
-
- <div class="bd-container">
- <div class="bd-container__inner bd-page-width">
-
-
-
- <dialog id="pst-primary-sidebar-modal"></dialog>
- <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
-
-
-
- <div class="sidebar-header-items sidebar-primary__section">
-
-
- <div class="sidebar-header-items__center">
-
-
-
- <div class="navbar-item">
-<nav>
- <ul class="bd-navbar-elements navbar-nav">
-
-<li class="nav-item current active">
- <a class="nav-link nav-internal" href="../index.html">
- User Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
- Contributor Guide
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
- Comparison with Gluten
- </a>
-</li>
-
-
-<li class="nav-item ">
- <a class="nav-link nav-internal" href="../../asf/index.html">
- ASF Links
- </a>
-</li>
-
- </ul>
-</nav></div>
-
-
- </div>
-
-
-
- <div class="sidebar-header-items__end">
-
- <div class="navbar-item"><ul class="navbar-icon-links"
- aria-label="Icon Links">
- <li class="nav-item">
-
-
-
-
-
-
-
-
- <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
- <span class="sr-only">GitHub</span></a>
- </li>
-</ul></div>
-
- <div class="navbar-item">
-
-<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
- <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
- <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
- <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
-</button></div>
-
- </div>
-
- </div>
-
- <div class="sidebar-primary-items__start sidebar-primary__section">
- <div class="sidebar-primary-item"><!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
- <div class="bd-toc-item active">
- <p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
-<ul class="current">
-<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.12.x</a><ul class="current">
-<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
-<li class="toctree-l3"><a class="reference internal" href="source.html">Building From Source</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datasources.html">Supported Data Sources</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datatypes.html">Supported Data Types</a></li>
-<li class="toctree-l3 current"><a class="current reference internal" href="#">Supported Operators</a></li>
-<li class="toctree-l3"><a class="reference internal" href="expressions.html">Supported Expressions</a></li>
-<li class="toctree-l3"><a class="reference internal" href="configs.html">Configuration Settings</a></li>
-<li class="toctree-l3"><a class="reference internal" href="compatibility.html">Compatibility Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="tuning.html">Tuning Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="metrics.html">Metrics Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
-<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
-</ul>
-</li>
-</ul>
-</li>
-<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../about/gluten_comparison.html">Comparison with Gluten</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../asf/index.html">ASF Links</a></li>
-</ul>
-
- </div>
-</nav>
-</div>
- </div>
-
-
- <div class="sidebar-primary-items__end sidebar-primary__section">
- <div class="sidebar-primary-item">
-<div id="ethical-ad-placement"
- class="flat"
- data-ea-publisher="readthedocs"
- data-ea-type="readthedocs-sidebar"
- data-ea-manual="true">
-</div></div>
- </div>
-
-
- </div>
-
- <main id="main-content" class="bd-main" role="main">
-
-
- <div class="bd-content">
- <div class="bd-article-container">
-
- <div class="bd-header-article d-print-none">
-<div class="header-article-items header-article__inner">
-
- <div class="header-article-items__start">
-
- <div class="header-article-item">
-
-<nav aria-label="Breadcrumb" class="d-print-none">
- <ul class="bd-breadcrumbs">
-
- <li class="breadcrumb-item breadcrumb-home">
- <a href="../../index.html" class="nav-link" aria-label="Home">
- <i class="fa-solid fa-home"></i>
- </a>
- </li>
-
- <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Comet User Guide</a></li>
-
-
- <li class="breadcrumb-item"><a href="index.html" class="nav-link">Comet 0.12.0 User Guide</a></li>
-
- <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Supported Spark Operators</span></li>
- </ul>
-</nav>
-</div>
-
- </div>
-
-
-</div>
-</div>
-
-
-
-
-<div id="searchbox"></div>
- <article class="bd-article">
-
- <!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<div class="admonition warning">
-<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
-</div>
-<section id="supported-spark-operators">
-<h1>Supported Spark Operators<a class="headerlink" href="#supported-spark-operators" title="Link to this heading">#</a></h1>
-<p>The following Spark operators are currently replaced with native versions. Query stages that contain any operators
-not supported by Comet will fall back to regular Spark execution.</p>
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Operator</p></th>
-<th class="head"><p>Spark-Compatible?</p></th>
-<th class="head"><p>Compatibility Notes</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p>BatchScanExec</p></td>
-<td><p>Yes</p></td>
-<td><p>Supports Parquet files and Apache Iceberg Parquet scans. See the <a class="reference internal" href="compatibility.html"><span class="std std-doc">Comet Compatibility Guide</span></a> for more information.</p></td>
-</tr>
-<tr class="row-odd"><td><p>BroadcastExchangeExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>BroadcastHashJoinExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>ExpandExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>FileSourceScanExec</p></td>
-<td><p>Yes</p></td>
-<td><p>Supports Parquet files. See the <a class="reference internal" href="compatibility.html"><span class="std std-doc">Comet Compatibility Guide</span></a> for more information.</p></td>
-</tr>
-<tr class="row-odd"><td><p>FilterExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>GlobalLimitExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>HashAggregateExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>LocalLimitExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>LocalTableScanExec</p></td>
-<td><p>No</p></td>
-<td><p>Experimental and disabled by default.</p></td>
-</tr>
-<tr class="row-even"><td><p>ObjectHashAggregateExec</p></td>
-<td><p>Yes</p></td>
-<td><p>Supports a limited number of aggregates, such as <code class="docutils literal notranslate"><span class="pre">bloom_filter_agg</span></code>.</p></td>
-</tr>
-<tr class="row-odd"><td><p>ProjectExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>ShuffleExchangeExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>ShuffledHashJoinExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>SortExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>SortMergeJoinExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-even"><td><p>UnionExec</p></td>
-<td><p>Yes</p></td>
-<td><p></p></td>
-</tr>
-<tr class="row-odd"><td><p>WindowExec</p></td>
-<td><p>No</p></td>
-<td><p>Disabled by default due to known correctness issues.</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-</section>
-
-
- </article>
-
-
-
-
-
- <footer class="prev-next-footer d-print-none">
-
-<div class="prev-next-area">
- <a class="left-prev"
- href="datatypes.html"
- title="previous page">
- <i class="fa-solid fa-angle-left"></i>
- <div class="prev-next-info">
- <p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Supported Spark Data Types</p>
- </div>
- </a>
- <a class="right-next"
- href="expressions.html"
- title="next page">
- <div class="prev-next-info">
- <p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Supported Spark Expressions</p>
- </div>
- <i class="fa-solid fa-angle-right"></i>
- </a>
-</div>
- </footer>
-
- </div>
-
-
-
-
- </div>
- <footer class="bd-footer-content">
-
- </footer>
-
- </main>
- </div>
- </div>
-
- <!-- Scripts loaded after <body> so the DOM is not blocked -->
- <script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
-<script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
-
-<!-- Based on pydata_sphinx_theme/footer.html -->
-<footer class="footer mt-5 mt-md-0">
- <div class="container">
-
- <div class="footer-item">
- <p>Apache DataFusion, Apache DataFusion Comet, Apache, the Apache feather logo, and the Apache DataFusion project logo</p>
- <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
- </div>
- </div>
-</footer>
-
-
- </body>
-</html>
\ No newline at end of file
diff --git a/user-guide/0.13/compatibility.html b/user-guide/0.13/compatibility.html
index 6ae6e4879..f34abfec3 100644
--- a/user-guide/0.13/compatibility.html
+++ b/user-guide/0.13/compatibility.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="compatibility-guide">
<h1>Compatibility Guide<a class="headerlink" href="#compatibility-guide" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/configs.html b/user-guide/0.13/configs.html
index 70a2c1265..50f641b7f 100644
--- a/user-guide/0.13/configs.html
+++ b/user-guide/0.13/configs.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="comet-configuration-settings">
<h1>Comet Configuration Settings<a class="headerlink" href="#comet-configuration-settings" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/datasources.html b/user-guide/0.13/datasources.html
index d1351b359..849b8c7cd 100644
--- a/user-guide/0.13/datasources.html
+++ b/user-guide/0.13/datasources.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="supported-spark-data-sources">
<h1>Supported Spark Data Sources<a class="headerlink" href="#supported-spark-data-sources" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/datatypes.html b/user-guide/0.13/datatypes.html
index 9f09f0bb1..615e58616 100644
--- a/user-guide/0.13/datatypes.html
+++ b/user-guide/0.13/datatypes.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="supported-spark-data-types">
<h1>Supported Spark Data Types<a class="headerlink" href="#supported-spark-data-types" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/expressions.html b/user-guide/0.13/expressions.html
index 4f71e6e40..fbd55aa29 100644
--- a/user-guide/0.13/expressions.html
+++ b/user-guide/0.13/expressions.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="supported-spark-expressions">
<h1>Supported Spark Expressions<a class="headerlink" href="#supported-spark-expressions" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/iceberg.html b/user-guide/0.13/iceberg.html
index 50f176c09..357791b1b 100644
--- a/user-guide/0.13/iceberg.html
+++ b/user-guide/0.13/iceberg.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="accelerating-apache-iceberg-parquet-scans-using-comet-experimental">
<h1>Accelerating Apache Iceberg Parquet Scans using Comet (Experimental)<a class="headerlink" href="#accelerating-apache-iceberg-parquet-scans-using-comet-experimental" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/index.html b/user-guide/0.13/index.html
index 3a50d80b2..358c01e6b 100644
--- a/user-guide/0.13/index.html
+++ b/user-guide/0.13/index.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">0.13.x</a><ul>
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
diff --git a/user-guide/0.13/installation.html b/user-guide/0.13/installation.html
index 052f0c246..b642df4f1 100644
--- a/user-guide/0.13/installation.html
+++ b/user-guide/0.13/installation.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="installing-datafusion-comet">
<h1>Installing DataFusion Comet<a class="headerlink" href="#installing-datafusion-comet" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/kubernetes.html b/user-guide/0.13/kubernetes.html
index f436bd15f..a4d255be5 100644
--- a/user-guide/0.13/kubernetes.html
+++ b/user-guide/0.13/kubernetes.html
@@ -65,7 +65,7 @@ under the License.
<script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Comet 0.12.0 User Guide" href="../0.12/index.html" />
+ <link rel="next" title="Comet Contributor Guide" href="../../contributor-guide/index.html" />
<link rel="prev" title="Accelerating Apache Iceberg Parquet Scans using Comet (Experimental)" href="iceberg.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3 current"><a class="current reference internal" href="#">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="comet-kubernetes-support">
<h1>Comet Kubernetes Support<a class="headerlink" href="#comet-kubernetes-support" title="Link to this heading">#</a></h1>
@@ -593,11 +593,11 @@ Events:
</div>
</a>
<a class="right-next"
- href="../0.12/index.html"
+ href="../../contributor-guide/index.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Comet 0.12.0 User Guide</p>
+ <p class="prev-next-title">Comet Contributor Guide</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
diff --git a/user-guide/0.13/metrics.html b/user-guide/0.13/metrics.html
index 44881f4f4..8665a1b4c 100644
--- a/user-guide/0.13/metrics.html
+++ b/user-guide/0.13/metrics.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="comet-metrics">
<h1>Comet Metrics<a class="headerlink" href="#comet-metrics" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/operators.html b/user-guide/0.13/operators.html
index cb01fa5f7..18523c705 100644
--- a/user-guide/0.13/operators.html
+++ b/user-guide/0.13/operators.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="supported-spark-operators">
<h1>Supported Spark Operators<a class="headerlink" href="#supported-spark-operators" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/source.html b/user-guide/0.13/source.html
index b3d8a3710..94e64caa1 100644
--- a/user-guide/0.13/source.html
+++ b/user-guide/0.13/source.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="building-comet-from-source">
<h1>Building Comet From Source<a class="headerlink" href="#building-comet-from-source" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.13/tuning.html b/user-guide/0.13/tuning.html
index 6155e6908..77b031280 100644
--- a/user-guide/0.13/tuning.html
+++ b/user-guide/0.13/tuning.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.13.x</a><ul class="current">
@@ -369,7 +370,6 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="comet-tuning-guide">
<h1>Comet Tuning Guide<a class="headerlink" href="#comet-tuning-guide" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/compatibility.html b/user-guide/0.14/compatibility.html
index c596489b5..31976d7c2 100644
--- a/user-guide/0.14/compatibility.html
+++ b/user-guide/0.14/compatibility.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="compatibility-guide">
<h1>Compatibility Guide<a class="headerlink" href="#compatibility-guide" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/configs.html b/user-guide/0.14/configs.html
index a6a068e2b..0394201dd 100644
--- a/user-guide/0.14/configs.html
+++ b/user-guide/0.14/configs.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="comet-configuration-settings">
<h1>Comet Configuration Settings<a class="headerlink" href="#comet-configuration-settings" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/datasources.html b/user-guide/0.14/datasources.html
index 400b34337..544566674 100644
--- a/user-guide/0.14/datasources.html
+++ b/user-guide/0.14/datasources.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="supported-spark-data-sources">
<h1>Supported Spark Data Sources<a class="headerlink" href="#supported-spark-data-sources" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/datatypes.html b/user-guide/0.14/datatypes.html
index 2b9b56882..8ab3aca8c 100644
--- a/user-guide/0.14/datatypes.html
+++ b/user-guide/0.14/datatypes.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="supported-spark-data-types">
<h1>Supported Spark Data Types<a class="headerlink" href="#supported-spark-data-types" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/expressions.html b/user-guide/0.14/expressions.html
index b5849f166..faf06d6e3 100644
--- a/user-guide/0.14/expressions.html
+++ b/user-guide/0.14/expressions.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="supported-spark-expressions">
<h1>Supported Spark Expressions<a class="headerlink" href="#supported-spark-expressions" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/iceberg.html b/user-guide/0.14/iceberg.html
index 74848a107..75f59ddec 100644
--- a/user-guide/0.14/iceberg.html
+++ b/user-guide/0.14/iceberg.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="accelerating-apache-iceberg-parquet-scans-using-comet-experimental">
<h1>Accelerating Apache Iceberg Parquet Scans using Comet (Experimental)<a class="headerlink" href="#accelerating-apache-iceberg-parquet-scans-using-comet-experimental" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/index.html b/user-guide/0.14/index.html
index 15f3c1165..b4f14e54e 100644
--- a/user-guide/0.14/index.html
+++ b/user-guide/0.14/index.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">0.14.x</a><ul>
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
diff --git a/user-guide/0.14/installation.html b/user-guide/0.14/installation.html
index f32543786..637cb5b0c 100644
--- a/user-guide/0.14/installation.html
+++ b/user-guide/0.14/installation.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3 current"><a class="current reference internal" href="#">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="installing-datafusion-comet">
<h1>Installing DataFusion Comet<a class="headerlink" href="#installing-datafusion-comet" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/kubernetes.html b/user-guide/0.14/kubernetes.html
index 9ec0c2265..fd09f636b 100644
--- a/user-guide/0.14/kubernetes.html
+++ b/user-guide/0.14/kubernetes.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="comet-kubernetes-support">
<h1>Comet Kubernetes Support<a class="headerlink" href="#comet-kubernetes-support" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/metrics.html b/user-guide/0.14/metrics.html
index d87e99897..acbf377a0 100644
--- a/user-guide/0.14/metrics.html
+++ b/user-guide/0.14/metrics.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="comet-metrics">
<h1>Comet Metrics<a class="headerlink" href="#comet-metrics" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/operators.html b/user-guide/0.14/operators.html
index ea51ba55a..aa2ed2372 100644
--- a/user-guide/0.14/operators.html
+++ b/user-guide/0.14/operators.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="supported-spark-operators">
<h1>Supported Spark Operators<a class="headerlink" href="#supported-spark-operators" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/source.html b/user-guide/0.14/source.html
index b4c6e46a2..a48689469 100644
--- a/user-guide/0.14/source.html
+++ b/user-guide/0.14/source.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="building-comet-from-source">
<h1>Building Comet From Source<a class="headerlink" href="#building-comet-from-source" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.14/tuning.html b/user-guide/0.14/tuning.html
index 436dfd4cd..e16f0907a 100644
--- a/user-guide/0.14/tuning.html
+++ b/user-guide/0.14/tuning.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.15/index.html">0.15.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.14.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
@@ -369,7 +370,6 @@ under the License.
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -460,7 +460,7 @@ under the License.
-->
<div class="admonition warning">
<p class="admonition-title">Warning</p>
-<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.15.0.</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
</div>
<section id="comet-tuning-guide">
<h1>Comet Tuning Guide<a class="headerlink" href="#comet-tuning-guide" title="Link to this heading">#</a></h1>
diff --git a/user-guide/0.15/compatibility.html b/user-guide/0.15/compatibility.html
index 105297ba5..fdd74151f 100644
--- a/user-guide/0.15/compatibility.html
+++ b/user-guide/0.15/compatibility.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.15.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
<li class="toctree-l3"><a class="reference internal" href="source.html">Building From Source</a></li>
@@ -369,7 +370,6 @@ under the License.
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -458,6 +458,10 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
+<div class="admonition warning">
+<p class="admonition-title">Warning</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
+</div>
<section id="compatibility-guide">
<h1>Compatibility Guide<a class="headerlink" href="#compatibility-guide" title="Link to this heading">#</a></h1>
<p>Comet aims to provide consistent results with the version of Apache Spark that is being used.</p>
diff --git a/user-guide/0.15/configs.html b/user-guide/0.15/configs.html
index 7f856376d..ef641dee9 100644
--- a/user-guide/0.15/configs.html
+++ b/user-guide/0.15/configs.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.15.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
<li class="toctree-l3"><a class="reference internal" href="source.html">Building From Source</a></li>
@@ -369,7 +370,6 @@ under the License.
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -458,6 +458,10 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
+<div class="admonition warning">
+<p class="admonition-title">Warning</p>
+<p>This is <strong>out-of-date</strong> documentation. The latest Comet release is version 0.16.0.</p>
+</div>
<section id="comet-configuration-settings">
<h1>Comet Configuration Settings<a class="headerlink" href="#comet-configuration-settings" title="Link to this heading">#</a></h1>
<p>Comet provides the following configuration settings.</p>
diff --git a/user-guide/0.15/datasources.html b/user-guide/0.15/datasources.html
index 6013efce0..e6100d0fb 100644
--- a/user-guide/0.15/datasources.html
+++ b/user-guide/0.15/datasources.html
@@ -351,7 +351,8 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Index</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">User Guide</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.16.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/index.html">0.17.0-SNAPSHOT</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../0.16/index.html">0.16.x</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">0.15.x</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="installation.html">Installing Comet</a></li>
<li class="toctree-l3"><a class="reference internal" href="source.html">Building From Source</a></li>
@@ -369,7 +370,6 @@ under the License.
</li>
<li class="toctree-l2"><a class="reference internal" href="../0.14/index.html">0.14.x</a></li>
<li class="toctree-l2"><a class="reference internal" href="../0.13/index.html">0.13.x</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../0.12/index.html">0.12.x</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a></li>
@@ -458,6 +458,10 @@ under the License.
specific language governing permissions and limitations
under the License.
-->
... 4403 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org