You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2025/01/14 03:21:04 UTC

(datafusion-comet) branch asf-site updated: Publish built docs triggered by 9fe5420edb529ebfa8b45c4ddb00b7dc9c0ca199

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


The following commit(s) were added to refs/heads/asf-site by this push:
     new 6a8ef753 Publish built docs triggered by 9fe5420edb529ebfa8b45c4ddb00b7dc9c0ca199
6a8ef753 is described below

commit 6a8ef75381ed2d5f79d80d7990327f0c2b541ed8
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Tue Jan 14 03:20:59 2025 +0000

    Publish built docs triggered by 9fe5420edb529ebfa8b45c4ddb00b7dc9c0ca199
---
 .../comet-tpch.json                                | 219 ---------------------
 .../spark-tpch.json                                | 203 -------------------
 .../comet-tpch.json                                | 209 ++++++++++++++++++++
 .../spark-tpch.json                                | 185 +++++++++++++++++
 _images/tpch_allqueries.png                        | Bin 29251 -> 0 bytes
 _images/tpch_queries_compare.png                   | Bin 34357 -> 0 bytes
 _images/tpch_queries_speedup_abs.png               | Bin 37579 -> 0 bytes
 _images/tpch_queries_speedup_rel.png               | Bin 48051 -> 0 bytes
 .../benchmark-results/tpc-h.md.txt                 |  14 +-
 _sources/contributor-guide/benchmarking.md.txt     |  14 +-
 _sources/user-guide/configs.md.txt                 |   1 +
 .../benchmark-results/0.5.0/tpch_allqueries.png    | Bin 0 -> 28138 bytes
 .../0.5.0/tpch_queries_compare.png                 | Bin 0 -> 33059 bytes
 .../0.5.0/tpch_queries_speedup_abs.png             | Bin 0 -> 38156 bytes
 .../0.5.0/tpch_queries_speedup_rel.png             | Bin 0 -> 48224 bytes
 contributor-guide/benchmark-results/tpc-h.html     |  14 +-
 contributor-guide/benchmarking.html                |  27 ++-
 searchindex.js                                     |   2 +-
 user-guide/configs.html                            |  68 ++++---
 19 files changed, 483 insertions(+), 473 deletions(-)

diff --git a/_downloads/180af7f5ca0d6ea4521d87c6058b110f/comet-tpch.json b/_downloads/180af7f5ca0d6ea4521d87c6058b110f/comet-tpch.json
deleted file mode 100644
index 25404f87..00000000
--- a/_downloads/180af7f5ca0d6ea4521d87c6058b110f/comet-tpch.json
+++ /dev/null
@@ -1,219 +0,0 @@
-{
-    "engine": "datafusion-comet",
-    "benchmark": "tpch",
-    "data_path": "/mnt/bigdata/tpch/sf100/",
-    "query_path": "/tpch",
-    "spark_conf": {
-        "spark.comet.cast.allowIncompatible": "true",
-        "spark.kubernetes.memoryOverheadFactor": "0.4",
-        "spark.kubernetes.driver.volumes.hostPath.data.mount.path": "/mnt/bigdata",
-        "spark.driver.host": "tpcbench-py-a08b54934067a8b3-driver-svc.default.svc",
-        "spark.memory.offHeap.size": "32g",
-        "spark.master": "k8s://https://kube1:6443",
-        "spark.kubernetes.driver.volumes.hostPath.data.options.path": "/mnt/bigdata",
-        "spark.comet.exec.all.enabled": "true",
-        "spark.driver.bindAddress": "10.42.0.10",
-        "spark.eventLog.enabled": "false",
-        "spark.kubernetes.container.image.pullPolicy": "Always",
-        "spark.kubernetes.submitInDriver": "true",
-        "spark.app.submitTime": "1731952357631",
-        "spark.sql.warehouse.dir": "file:/opt/spark/work-dir/spark-warehouse",
-        "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
-        "spark.jars": "local:/opt/spark/jars/comet-spark-spark3.4_2.12-0.4.0-SNAPSHOT.jar",
-        "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
-        "spark.comet.exec.enabled": "true",
-        "spark.executor.id": "driver",
-        "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly": "false",
-        "spark.comet.exec.shuffle.mode": "auto",
-        "spark.driver.memory": "8G",
-        "spark.driver.port": "7078",
-        "spark.sql.extensions": "org.apache.comet.CometSparkSessionExtensions",
-        "spark.kubernetes.driver.volumes.hostPath.data.options.readOnly": "false",
-        "spark.kubernetes.executor.podNamePrefix": "comet-benchmark-derived-from-tpch-42d900934067b351",
-        "spark.kubernetes.executor.volumes.hostPath.data.mount.path": "/mnt/bigdata",
-        "spark.kubernetes.driver.container.image": "woody:5000/spark-tpcbench:comet",
-        "spark.executor.instances": "1",
-        "spark.kubernetes.driver.pod.name": "tpcbench-py-a08b54934067a8b3-driver",
-        "spark.executor.extraClassPath": "local:///opt/spark/jars/comet-spark-spark3.4_2.12-0.4.0-SNAPSHOT.jar",
-        "spark.comet.explain.native.enabled": "false",
-        "spark.executor.memoryOverhead": "4914M",
-        "spark.app.initial.jar.urls": "file:/opt/spark/jars/comet-spark-spark3.4_2.12-0.4.0-SNAPSHOT.jar",
-        "spark.kubernetes.authenticate.driver.serviceAccountName": "spark",
-        "spark.kubernetes.executor.container.image": "woody:5000/spark-tpcbench:comet",
-        "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName": "benchmark-results-pvc",
-        "spark.comet.exec.replaceSortMergeJoin": "true",
-        "spark.comet.exec.shuffle.enabled": "true",
-        "spark.memory.offHeap.enabled": "true",
-        "spark.app.name": "comet benchmark derived from tpch",
-        "spark.executor.memory": "16G",
-        "spark.serializer.objectStreamReset": "100",
-        "spark.submit.deployMode": "client",
-        "spark.app.id": "spark-71d7e005c63b4f7a9a54d11eaa5d99d4",
-        "spark.executor.cores": "8",
-        "spark.driver.blockManager.port": "7079",
-        "spark.repl.local.jars": "local:///opt/spark/jars/comet-spark-spark3.4_2.12-0.4.0-SNAPSHOT.jar",
-        "spark.driver.extraClassPath": "local:///opt/spark/jars/comet-spark-spark3.4_2.12-0.4.0-SNAPSHOT.jar",
-        "spark.plugins": "org.apache.spark.CometPlugin",
-        "spark.app.startTime": "1731952357848",
-        "spark.kubernetes.resource.type": "python",
-        "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path": "/mnt/benchmark-results",
-        "spark.rdd.compress": "True",
-        "spark.kubernetes.executor.volumes.hostPath.data.options.path": "/mnt/bigdata",
-        "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
-        "spark.submit.pyFiles": ""
-    },
-    "1": [
-        12.675217866897583,
-        10.69823956489563,
-        10.959712743759155,
-        11.049235582351685,
-        11.25637674331665
-    ],
-    "2": [
-        7.2883970737457275,
-        6.955080509185791,
-        6.996195316314697,
-        7.001482725143433,
-        6.981860399246216
-    ],
-    "3": [
-        17.389552116394043,
-        17.815346479415894,
-        18.098077297210693,
-        18.250796794891357,
-        18.289159297943115
-    ],
-    "4": [
-        9.9054114818573,
-        9.76110053062439,
-        9.778793573379517,
-        9.659870386123657,
-        9.890227794647217
-    ],
-    "5": [
-        29.422667026519775,
-        29.41081738471985,
-        29.542785167694092,
-        29.379701375961304,
-        30.072590589523315
-    ],
-    "6": [
-        2.591843843460083,
-        2.5433740615844727,
-        2.486654758453369,
-        2.5132291316986084,
-        2.507908821105957
-    ],
-    "7": [
-        17.24040937423706,
-        17.355087280273438,
-        17.514805555343628,
-        17.43350887298584,
-        17.35370373725891
-    ],
-    "8": [
-        29.68608021736145,
-        29.521278858184814,
-        29.879571437835693,
-        29.99302840232849,
-        30.18179965019226
-    ],
-    "9": [
-        44.744760274887085,
-        45.07009744644165,
-        45.67892384529114,
-        45.49783205986023,
-        46.2200129032135
-    ],
-    "10": [
-        16.815892696380615,
-        16.816420078277588,
-        16.754045486450195,
-        16.916579008102417,
-        16.823299169540405
-    ],
-    "11": [
-        5.107349395751953,
-        4.9230499267578125,
-        4.9613683223724365,
-        4.954375743865967,
-        4.949444770812988
-    ],
-    "12": [
-        5.476846694946289,
-        5.373950719833374,
-        5.399810791015625,
-        5.373565196990967,
-        5.389242172241211
-    ],
-    "13": [
-        11.168701410293579,
-        11.108731031417847,
-        11.159045219421387,
-        10.97795581817627,
-        11.064870119094849
-    ],
-    "14": [
-        5.262895584106445,
-        5.161571502685547,
-        5.15362286567688,
-        5.113251209259033,
-        5.200662136077881
-    ],
-    "15": [
-        9.296111345291138,
-        9.049000263214111,
-        8.952059745788574,
-        9.111369371414185,
-        9.09269094467163
-    ],
-    "16": [
-        6.102790832519531,
-        5.706904411315918,
-        5.573681831359863,
-        5.436795473098755,
-        5.416778802871704
-    ],
-    "17": [
-        34.57182478904724,
-        34.73775792121887,
-        34.8440580368042,
-        35.2716920375824,
-        35.55662488937378
-    ],
-    "18": [
-        32.07862377166748,
-        31.731548070907593,
-        31.748567819595337,
-        31.76587414741516,
-        31.907649993896484
-    ],
-    "19": [
-        6.052667140960693,
-        5.940260887145996,
-        5.960237264633179,
-        6.020690441131592,
-        6.060962677001953
-    ],
-    "20": [
-        10.794468641281128,
-        10.485475778579712,
-        10.518470525741577,
-        10.661662101745605,
-        10.678892850875854
-    ],
-    "21": [
-        47.43690896034241,
-        46.92784881591797,
-        46.689401626586914,
-        52.07501554489136,
-        53.362786054611206
-    ],
-    "22": [
-        4.79883337020874,
-        4.179429054260254,
-        4.191235303878784,
-        4.875058889389038,
-        5.055111885070801
-    ]
-}
\ No newline at end of file
diff --git a/_downloads/9e848a9cc1377847bac6c7c280c15083/spark-tpch.json b/_downloads/9e848a9cc1377847bac6c7c280c15083/spark-tpch.json
deleted file mode 100644
index f0a080aa..00000000
--- a/_downloads/9e848a9cc1377847bac6c7c280c15083/spark-tpch.json
+++ /dev/null
@@ -1,203 +0,0 @@
-{
-    "engine": "datafusion-comet",
-    "benchmark": "tpch",
-    "data_path": "/mnt/bigdata/tpch/sf100/",
-    "query_path": "/tpch",
-    "spark_conf": {
-        "spark.kubernetes.executor.podNamePrefix": "spark-benchmark-derived-from-tpch-91eee2934092023f",
-        "spark.kubernetes.memoryOverheadFactor": "0.4",
-        "spark.kubernetes.authenticate.driver.serviceAccountName": "spark",
-        "spark.kubernetes.driver.volumes.hostPath.data.mount.path": "/mnt/bigdata",
-        "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName": "benchmark-results-pvc",
-        "spark.memory.offHeap.enabled": "true",
-        "spark.app.startTime": "1731955130576",
-        "spark.app.id": "spark-35db63a6dc9b42c79428c048244d508f",
-        "spark.master": "k8s://https://kube1:6443",
-        "spark.memory.offHeap.size": "48g",
-        "spark.kubernetes.driver.pod.name": "tpcbench-py-bca54a934091f6cd-driver",
-        "spark.kubernetes.executor.container.image": "apache/spark:3.4.2",
-        "spark.kubernetes.driver.container.image": "woody:5000/spark-tpcbench:spark",
-        "spark.executor.memory": "16G",
-        "spark.serializer.objectStreamReset": "100",
-        "spark.submit.deployMode": "client",
-        "spark.driver.host": "tpcbench-py-bca54a934091f6cd-driver-svc.default.svc",
-        "spark.kubernetes.driver.volumes.hostPath.data.options.path": "/mnt/bigdata",
-        "spark.eventLog.enabled": "false",
-        "spark.kubernetes.container.image.pullPolicy": "Always",
-        "spark.executor.cores": "8",
-        "spark.kubernetes.submitInDriver": "true",
-        "spark.sql.warehouse.dir": "file:/opt/spark/work-dir/spark-warehouse",
-        "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
-        "spark.driver.blockManager.port": "7079",
-        "spark.driver.bindAddress": "10.42.0.12",
-        "spark.executor.id": "driver",
-        "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly": "false",
-        "spark.driver.memory": "8G",
-        "spark.driver.port": "7078",
-        "spark.kubernetes.driver.volumes.hostPath.data.options.readOnly": "false",
-        "spark.kubernetes.resource.type": "python",
-        "spark.kubernetes.executor.volumes.hostPath.data.mount.path": "/mnt/bigdata",
-        "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path": "/mnt/benchmark-results",
-        "spark.rdd.compress": "True",
-        "spark.app.name": "spark benchmark derived from tpch",
-        "spark.kubernetes.executor.volumes.hostPath.data.options.path": "/mnt/bigdata",
-        "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
-        "spark.executor.instances": "1",
-        "spark.app.submitTime": "1731955130325",
-        "spark.submit.pyFiles": ""
-    },
-    "1": [
-        76.3459792137146,
-        77.43206644058228,
-        77.43688368797302,
-        77.57843589782715,
-        77.36109328269958
-    ],
-    "2": [
-        11.60047721862793,
-        12.094002485275269,
-        11.385731935501099,
-        11.742280006408691,
-        10.831515073776245
-    ],
-    "3": [
-        23.393443822860718,
-        23.679790496826172,
-        23.2339928150177,
-        23.296000957489014,
-        23.57963228225708
-    ],
-    "4": [
-        17.686111450195312,
-        17.663344860076904,
-        17.97491693496704,
-        17.99352526664734,
-        17.13844132423401
-    ],
-    "5": [
-        47.576931953430176,
-        47.90366792678833,
-        47.885528802871704,
-        47.82782196998596,
-        47.35511922836304
-    ],
-    "6": [
-        3.1415493488311768,
-        3.0869905948638916,
-        3.0410678386688232,
-        3.0086350440979004,
-        3.006296157836914
-    ],
-    "7": [
-        20.144665241241455,
-        20.645880699157715,
-        20.30326247215271,
-        20.52312994003296,
-        20.44648313522339
-    ],
-    "8": [
-        31.149712800979614,
-        31.346466541290283,
-        31.87547278404236,
-        31.968525409698486,
-        31.624362230300903
-    ],
-    "9": [
-        70.20270848274231,
-        73.088698387146,
-        71.15446996688843,
-        70.36094760894775,
-        71.0561454296112
-    ],
-    "10": [
-        19.035035133361816,
-        19.235825538635254,
-        19.032540559768677,
-        18.992990732192993,
-        18.960989236831665
-    ],
-    "11": [
-        12.353803873062134,
-        12.383912324905396,
-        12.390283346176147,
-        12.384623289108276,
-        12.608469009399414
-    ],
-    "12": [
-        12.46756649017334,
-        12.692992210388184,
-        13.037216424942017,
-        14.199844360351562,
-        12.41795563697815
-    ],
-    "13": [
-        21.14127492904663,
-        21.24750590324402,
-        21.330607652664185,
-        21.19257688522339,
-        21.117241144180298
-    ],
-    "14": [
-        5.365051031112671,
-        5.244381904602051,
-        5.224266290664673,
-        5.279320478439331,
-        5.351390600204468
-    ],
-    "15": [
-        14.187044382095337,
-        14.210954904556274,
-        14.301335096359253,
-        14.133733034133911,
-        14.062178373336792
-    ],
-    "16": [
-        6.709480047225952,
-        6.949545383453369,
-        6.819269180297852,
-        6.797849178314209,
-        7.115732192993164
-    ],
-    "17": [
-        59.576388120651245,
-        60.451228857040405,
-        60.65900158882141,
-        60.63210201263428,
-        60.68089413642883
-    ],
-    "18": [
-        69.60278511047363,
-        70.1350531578064,
-        69.89858436584473,
-        70.29715323448181,
-        69.83147168159485
-    ],
-    "19": [
-        6.37906551361084,
-        6.309690952301025,
-        6.3234429359436035,
-        6.329152345657349,
-        6.348209381103516
-    ],
-    "20": [
-        9.532709121704102,
-        9.26326322555542,
-        9.232454776763916,
-        9.317341089248657,
-        9.209791898727417
-    ],
-    "21": [
-        65.31082701683044,
-        65.26856255531311,
-        64.04693102836609,
-        64.84621214866638,
-        64.52755498886108
-    ],
-    "22": [
-        8.907754182815552,
-        8.766010046005249,
-        8.830661535263062,
-        8.698703527450562,
-        8.652528524398804
-    ]
-}
\ No newline at end of file
diff --git a/_downloads/a57370497e8278995e02e50496e3d7d8/comet-tpch.json b/_downloads/a57370497e8278995e02e50496e3d7d8/comet-tpch.json
new file mode 100644
index 00000000..b1b3ae12
--- /dev/null
+++ b/_downloads/a57370497e8278995e02e50496e3d7d8/comet-tpch.json
@@ -0,0 +1,209 @@
+{
+    "engine": "datafusion-comet",
+    "benchmark": "tpch",
+    "data_path": "/mnt/bigdata/tpch/sf100/",
+    "query_path": "/home/andy/git/apache/datafusion-benchmarks/tpch/queries",
+    "spark_conf": {
+        "spark.comet.explain.native.enabled": "false",
+        "spark.eventLog.enabled": "true",
+        "spark.executor.extraClassPath": "/home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
+        "spark.comet.explainFallback.enabled": "false",
+        "spark.comet.exec.replaceSortMergeJoin": "true",
+        "spark.comet.exec.shuffle.enabled": "true",
+        "spark.memory.offHeap.enabled": "true",
+        "spark.comet.exec.shuffle.compression.level": "1",
+        "spark.executor.memory": "16g",
+        "spark.app.name": "comet benchmark derived from tpch",
+        "spark.comet.batchSize": "8192",
+        "spark.app.startTime": "1736802464855",
+        "spark.comet.exec.shuffle.fallbackToColumnar": "true",
+        "spark.serializer.objectStreamReset": "100",
+        "spark.driver.host": "10.0.0.118",
+        "spark.comet.exec.shuffle.enableFastEncoding": "true",
+        "spark.submit.deployMode": "client",
+        "spark.driver.port": "33103",
+        "spark.comet.scan.impl": "native_comet",
+        "spark.driver.extraClassPath": "/home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
+        "spark.executor.cores": "8",
+        "spark.comet.explain.verbose.enabled": "false",
+        "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
+        "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
+        "spark.comet.exec.enabled": "true",
+        "spark.sql.warehouse.dir": "file:/home/andy/git/personal/research/benchmarks/spark-standalone/spark-warehouse",
+        "spark.comet.scan.enabled": "true",
+        "spark.app.submitTime": "1736802464584",
+        "spark.executor.id": "driver",
+        "spark.master": "spark://woody:7077",
+        "spark.comet.exec.shuffle.mode": "auto",
+        "spark.sql.extensions": "org.apache.comet.CometSparkSessionExtensions",
+        "spark.driver.memory": "8G",
+        "spark.repl.local.jars": "file:///home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
+        "spark.app.initial.jar.urls": "spark://10.0.0.118:33103/jars/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
+        "spark.app.id": "app-20250113140745-0058",
+        "spark.rdd.compress": "True",
+        "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
+        "spark.executor.instances": "1",
+        "spark.cores.max": "8",
+        "spark.comet.enabled": "true",
+        "spark.submit.pyFiles": "",
+        "spark.comet.exec.sortMergeJoinWithJoinFilter.enabled": "false",
+        "spark.comet.exec.shuffle.compression.codec": "lz4",
+        "spark.jars": "file:///home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
+        "spark.memory.offHeap.size": "16g",
+        "spark.comet.columnar.shuffle.batch.size": "8192"
+    },
+    "1": [
+        12.59755539894104,
+        10.855465650558472,
+        11.160947799682617,
+        11.323237657546997,
+        11.410452365875244
+    ],
+    "2": [
+        6.155406475067139,
+        5.539891719818115,
+        5.698071002960205,
+        5.684133529663086,
+        5.742799758911133
+    ],
+    "3": [
+        16.097025156021118,
+        14.982890367507935,
+        14.998259544372559,
+        15.659432649612427,
+        15.878185749053955
+    ],
+    "4": [
+        10.319517850875854,
+        10.0553297996521,
+        10.136846780776978,
+        9.925675392150879,
+        10.140193462371826
+    ],
+    "5": [
+        26.09030055999756,
+        25.57556390762329,
+        26.102373600006104,
+        26.540887117385864,
+        26.162983655929565
+    ],
+    "6": [
+        2.691145658493042,
+        2.5986382961273193,
+        2.659151792526245,
+        2.6488683223724365,
+        2.6785433292388916
+    ],
+    "7": [
+        15.326677560806274,
+        15.57035493850708,
+        16.023503065109253,
+        16.015883207321167,
+        15.79127025604248
+    ],
+    "8": [
+        27.72478675842285,
+        27.45163321495056,
+        27.935590267181396,
+        27.86525869369507,
+        28.016165733337402
+    ],
+    "9": [
+        39.186867237091064,
+        39.73552465438843,
+        40.866581439971924,
+        40.73869442939758,
+        40.89244842529297
+    ],
+    "10": [
+        14.022773742675781,
+        14.476953029632568,
+        14.305155515670776,
+        14.187727451324463,
+        14.57831335067749
+    ],
+    "11": [
+        5.223851919174194,
+        4.722897291183472,
+        4.844727277755737,
+        4.803720474243164,
+        4.822873592376709
+    ],
+    "12": [
+        4.974349021911621,
+        5.013054132461548,
+        5.0682995319366455,
+        5.1071436405181885,
+        5.142468452453613
+    ],
+    "13": [
+        9.769477128982544,
+        9.743404626846313,
+        9.935744285583496,
+        9.966437339782715,
+        9.854998588562012
+    ],
+    "14": [
+        5.320314168930054,
+        5.26824426651001,
+        5.269179344177246,
+        5.322073698043823,
+        5.292902708053589
+    ],
+    "15": [
+        9.532674789428711,
+        9.520610570907593,
+        9.538906335830688,
+        9.553953886032104,
+        9.65409803390503
+    ],
+    "16": [
+        5.146467924118042,
+        4.716687440872192,
+        4.863113164901733,
+        4.725494384765625,
+        4.653785228729248
+    ],
+    "17": [
+        30.45087242126465,
+        30.785797119140625,
+        30.950777530670166,
+        31.04833745956421,
+        31.12831139564514
+    ],
+    "18": [
+        27.549716472625732,
+        27.610363960266113,
+        27.41417407989502,
+        27.633289098739624,
+        27.72838020324707
+    ],
+    "19": [
+        5.9813477993011475,
+        6.041543483734131,
+        6.087557554244995,
+        6.106397390365601,
+        6.011293888092041
+    ],
+    "20": [
+        10.53919005393982,
+        10.382107019424438,
+        10.370867729187012,
+        10.376642942428589,
+        10.48800802230835
+    ],
+    "21": [
+        42.36113142967224,
+        42.296979904174805,
+        42.56899857521057,
+        42.587459564208984,
+        42.86927652359009
+    ],
+    "22": [
+        3.755877733230591,
+        3.523585319519043,
+        3.5420711040496826,
+        3.605468273162842,
+        3.6084585189819336
+    ]
+}
\ No newline at end of file
diff --git a/_downloads/eb4015c41cfdc9c2fc70045fbf7f0c84/spark-tpch.json b/_downloads/eb4015c41cfdc9c2fc70045fbf7f0c84/spark-tpch.json
new file mode 100644
index 00000000..a82ea195
--- /dev/null
+++ b/_downloads/eb4015c41cfdc9c2fc70045fbf7f0c84/spark-tpch.json
@@ -0,0 +1,185 @@
+{
+    "engine": "datafusion-comet",
+    "benchmark": "tpch",
+    "data_path": "/mnt/bigdata/tpch/sf100/",
+    "query_path": "/home/andy/git/apache/datafusion-benchmarks/tpch/queries",
+    "spark_conf": {
+        "spark.eventLog.enabled": "true",
+        "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
+        "spark.sql.warehouse.dir": "file:/home/andy/git/personal/research/benchmarks/spark-standalone/spark-warehouse",
+        "spark.executor.id": "driver",
+        "spark.master": "spark://woody:7077",
+        "spark.driver.memory": "8G",
+        "spark.memory.offHeap.enabled": "true",
+        "spark.driver.port": "34999",
+        "spark.executor.memory": "16g",
+        "spark.app.id": "app-20250113143837-0059",
+        "spark.app.startTime": "1736804317132",
+        "spark.rdd.compress": "True",
+        "spark.app.name": "spark benchmark derived from tpch",
+        "spark.app.submitTime": "1736804316845",
+        "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
+        "spark.serializer.objectStreamReset": "100",
+        "spark.driver.host": "10.0.0.118",
+        "spark.executor.instances": "1",
+        "spark.cores.max": "8",
+        "spark.submit.pyFiles": "",
+        "spark.submit.deployMode": "client",
+        "spark.executor.cores": "8",
+        "spark.memory.offHeap.size": "16g"
+    },
+    "1": [
+        80.39240145683289,
+        79.9300479888916,
+        80.08533000946045,
+        79.95083689689636,
+        79.85243964195251
+    ],
+    "2": [
+        12.361650466918945,
+        12.305850267410278,
+        12.250919818878174,
+        11.97881293296814,
+        11.725982427597046
+    ],
+    "3": [
+        24.618537187576294,
+        24.87288188934326,
+        24.59152317047119,
+        24.218815803527832,
+        24.44655966758728
+    ],
+    "4": [
+        22.427627086639404,
+        20.657840728759766,
+        21.091227531433105,
+        21.05953359603882,
+        20.46818709373474
+    ],
+    "5": [
+        50.51570105552673,
+        49.615350008010864,
+        49.8210015296936,
+        49.847323179244995,
+        49.476715087890625
+    ],
+    "6": [
+        3.347008466720581,
+        3.21842360496521,
+        3.1940605640411377,
+        3.310222625732422,
+        3.24226713180542
+    ],
+    "7": [
+        21.848386764526367,
+        21.017245531082153,
+        21.077393054962158,
+        21.38729691505432,
+        21.2225079536438
+    ],
+    "8": [
+        32.984848499298096,
+        33.09590411186218,
+        32.93970465660095,
+        32.48174071311951,
+        32.516525983810425
+    ],
+    "9": [
+        76.0926308631897,
+        74.51611924171448,
+        74.34783029556274,
+        74.69494938850403,
+        74.43283152580261
+    ],
+    "10": [
+        19.82606053352356,
+        19.577861070632935,
+        19.701807498931885,
+        19.774757146835327,
+        19.246259689331055
+    ],
+    "11": [
+        12.777667760848999,
+        12.865848302841187,
+        12.908553838729858,
+        12.77224063873291,
+        12.639224767684937
+    ],
+    "12": [
+        13.610698223114014,
+        13.52022099494934,
+        13.459492206573486,
+        13.30645203590393,
+        13.632066011428833
+    ],
+    "13": [
+        22.276411771774292,
+        22.426157474517822,
+        22.358247995376587,
+        22.300530672073364,
+        22.17827558517456
+    ],
+    "14": [
+        5.936513662338257,
+        5.709101915359497,
+        5.788166046142578,
+        5.724794626235962,
+        5.8799731731414795
+    ],
+    "15": [
+        14.949971437454224,
+        14.810836553573608,
+        14.723956823348999,
+        14.827229738235474,
+        14.890477657318115
+    ],
+    "16": [
+        7.259998559951782,
+        6.874396324157715,
+        7.139331817626953,
+        7.010579586029053,
+        6.851199626922607
+    ],
+    "17": [
+        59.51790142059326,
+        61.10569620132446,
+        61.5246102809906,
+        62.90809988975525,
+        60.082926988601685
+    ],
+    "18": [
+        71.04480576515198,
+        71.47302889823914,
+        73.99175548553467,
+        70.6622040271759,
+        73.06630039215088
+    ],
+    "19": [
+        6.902673006057739,
+        7.26209568977356,
+        6.773332357406616,
+        6.810155868530273,
+        6.643052577972412
+    ],
+    "20": [
+        10.397570133209229,
+        9.828125,
+        9.871150493621826,
+        9.73079776763916,
+        9.764204502105713
+    ],
+    "21": [
+        66.58646297454834,
+        65.52946162223816,
+        65.2813880443573,
+        67.38516497612,
+        68.25078797340393
+    ],
+    "22": [
+        9.243720531463623,
+        9.1110098361969,
+        9.128684282302856,
+        9.107020854949951,
+        9.347189903259277
+    ]
+}
\ No newline at end of file
diff --git a/_images/tpch_allqueries.png b/_images/tpch_allqueries.png
deleted file mode 100644
index 24a3698b..00000000
Binary files a/_images/tpch_allqueries.png and /dev/null differ
diff --git a/_images/tpch_queries_compare.png b/_images/tpch_queries_compare.png
deleted file mode 100644
index 8d3223a9..00000000
Binary files a/_images/tpch_queries_compare.png and /dev/null differ
diff --git a/_images/tpch_queries_speedup_abs.png b/_images/tpch_queries_speedup_abs.png
deleted file mode 100644
index f8b3eeae..00000000
Binary files a/_images/tpch_queries_speedup_abs.png and /dev/null differ
diff --git a/_images/tpch_queries_speedup_rel.png b/_images/tpch_queries_speedup_rel.png
deleted file mode 100644
index d95b4150..00000000
Binary files a/_images/tpch_queries_speedup_rel.png and /dev/null differ
diff --git a/_sources/contributor-guide/benchmark-results/tpc-h.md.txt b/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
index 22854893..336deb7a 100644
--- a/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
+++ b/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
@@ -25,21 +25,21 @@ and we encourage you to run these benchmarks in your own environments.
 
 The tracking issue for improving TPC-H performance is [#391](https://github.com/apache/datafusion-comet/issues/391).
 
-![](../../_static/images/benchmark-results/0.4.0/tpch_allqueries.png)
+![](../../_static/images/benchmark-results/0.5.0-SNAPSHOT-2025-01-09/tpch_allqueries.png)
 
-Here is a breakdown showing relative performance of Spark, Comet, and DataFusion for each query.
+Here is a breakdown showing relative performance of Spark and Comet for each query.
 
-![](../../_static/images/benchmark-results/0.4.0/tpch_queries_compare.png)
+![](../../_static/images/benchmark-results/0.5.0-SNAPSHOT-2025-01-09/tpch_queries_compare.png)
 
 The following chart shows how much Comet currently accelerates each query from the benchmark in relative terms.
 
-![](../../_static/images/benchmark-results/0.4.0/tpch_queries_speedup_rel.png)
+![](../../_static/images/benchmark-results/0.5.0-SNAPSHOT-2025-01-09/tpch_queries_speedup_rel.png)
 
 The following chart shows how much Comet currently accelerates each query from the benchmark in absolute terms.
 
-![](../../_static/images/benchmark-results/0.4.0/tpch_queries_speedup_abs.png)
+![](../../_static/images/benchmark-results/0.5.0-SNAPSHOT-2025-01-09/tpch_queries_speedup_abs.png)
 
 The raw results of these benchmarks in JSON format is available here:
 
-- [Spark](0.4.0/spark-tpch.json)
-- [Comet](0.4.0/comet-tpch.json)
+- [Spark](0.5.0/spark-tpch.json)
+- [Comet](0.5.0/comet-tpch.json)
diff --git a/_sources/contributor-guide/benchmarking.md.txt b/_sources/contributor-guide/benchmarking.md.txt
index bd280f47..173d598a 100644
--- a/_sources/contributor-guide/benchmarking.md.txt
+++ b/_sources/contributor-guide/benchmarking.md.txt
@@ -49,6 +49,8 @@ $SPARK_HOME/bin/spark-submit \
 
 ## Running Benchmarks Against Apache Spark with Apache DataFusion Comet Enabled
 
+### TPC-H
+
 ```shell
 $SPARK_HOME/bin/spark-submit \
     --master $SPARK_MASTER \
@@ -58,15 +60,19 @@ $SPARK_HOME/bin/spark-submit \
     --conf spark.executor.cores=8 \
     --conf spark.cores.max=8 \
     --conf spark.memory.offHeap.enabled=true \
-    --conf spark.memory.offHeap.size=32g \
+    --conf spark.memory.offHeap.size=16g \
     --jars $COMET_JAR \
     --conf spark.driver.extraClassPath=$COMET_JAR \
     --conf spark.executor.extraClassPath=$COMET_JAR \
     --conf spark.plugins=org.apache.spark.CometPlugin \
     --conf spark.comet.cast.allowIncompatible=true \
     --conf spark.comet.exec.replaceSortMergeJoin=true \
-    --conf spark.comet.exec.shuffle.enabled=true \
     --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
+    --conf spark.comet.exec.shuffle.enabled=true \
+    --conf spark.comet.exec.shuffle.mode=auto \
+    --conf spark.comet.exec.shuffle.enableFastEncoding=true \
+    --conf spark.comet.exec.shuffle.fallbackToColumnar=true \
+    --conf spark.comet.exec.shuffle.compression.codec=lz4 \
     tpcbench.py \
     --benchmark tpch \
     --data /mnt/bigdata/tpch/sf100/ \
@@ -74,6 +80,10 @@ $SPARK_HOME/bin/spark-submit \
     --iterations 3
 ```
 
+### TPC-DS
+
+For TPC-DS, use `spark.comet.exec.replaceSortMergeJoin=false`.
+
 ## Current Benchmark Results
 
 - [Benchmarks derived from TPC-H](benchmark-results/tpc-h)
diff --git a/_sources/user-guide/configs.md.txt b/_sources/user-guide/configs.md.txt
index d78e6111..2bea501e 100644
--- a/_sources/user-guide/configs.md.txt
+++ b/_sources/user-guide/configs.md.txt
@@ -46,6 +46,7 @@ Comet provides the following configuration settings.
 | spark.comet.exec.filter.enabled | Whether to enable filter by default. | true |
 | spark.comet.exec.globalLimit.enabled | Whether to enable globalLimit by default. | true |
 | spark.comet.exec.hashJoin.enabled | Whether to enable hashJoin by default. | true |
+| spark.comet.exec.initCap.enabled | Whether to enable initCap by default. | false |
 | spark.comet.exec.localLimit.enabled | Whether to enable localLimit by default. | true |
 | spark.comet.exec.memoryPool | The type of memory pool to be used for Comet native execution. Available memory pool types are 'greedy', 'fair_spill', 'greedy_task_shared', 'fair_spill_task_shared', 'greedy_global' and 'fair_spill_global', By default, this config is 'greedy_task_shared'. | greedy_task_shared |
 | spark.comet.exec.project.enabled | Whether to enable project by default. | true |
diff --git a/_static/images/benchmark-results/0.5.0/tpch_allqueries.png b/_static/images/benchmark-results/0.5.0/tpch_allqueries.png
new file mode 100644
index 00000000..0855f7bf
Binary files /dev/null and b/_static/images/benchmark-results/0.5.0/tpch_allqueries.png differ
diff --git a/_static/images/benchmark-results/0.5.0/tpch_queries_compare.png b/_static/images/benchmark-results/0.5.0/tpch_queries_compare.png
new file mode 100644
index 00000000..c17c1af1
Binary files /dev/null and b/_static/images/benchmark-results/0.5.0/tpch_queries_compare.png differ
diff --git a/_static/images/benchmark-results/0.5.0/tpch_queries_speedup_abs.png b/_static/images/benchmark-results/0.5.0/tpch_queries_speedup_abs.png
new file mode 100644
index 00000000..aa8391cc
Binary files /dev/null and b/_static/images/benchmark-results/0.5.0/tpch_queries_speedup_abs.png differ
diff --git a/_static/images/benchmark-results/0.5.0/tpch_queries_speedup_rel.png b/_static/images/benchmark-results/0.5.0/tpch_queries_speedup_rel.png
new file mode 100644
index 00000000..1fce882e
Binary files /dev/null and b/_static/images/benchmark-results/0.5.0/tpch_queries_speedup_rel.png differ
diff --git a/contributor-guide/benchmark-results/tpc-h.html b/contributor-guide/benchmark-results/tpc-h.html
index fa05dc07..ceec9edd 100644
--- a/contributor-guide/benchmark-results/tpc-h.html
+++ b/contributor-guide/benchmark-results/tpc-h.html
@@ -331,17 +331,17 @@ under the License.
 data stored locally in Parquet format on NVMe storage. Performance characteristics will vary in different environments
 and we encourage you to run these benchmarks in your own environments.</p>
 <p>The tracking issue for improving TPC-H performance is <a class="reference external" href="https://github.com/apache/datafusion-comet/issues/391">#391</a>.</p>
-<p><img alt="" src="../../_images/tpch_allqueries.png" /></p>
-<p>Here is a breakdown showing relative performance of Spark, Comet, and DataFusion for each query.</p>
-<p><img alt="" src="../../_images/tpch_queries_compare.png" /></p>
+<p><img alt="" src="_static/images/benchmark-results/0.5.0-SNAPSHOT-2025-01-09/tpch_allqueries.png" /></p>
+<p>Here is a breakdown showing relative performance of Spark and Comet for each query.</p>
+<p><img alt="" src="_static/images/benchmark-results/0.5.0-SNAPSHOT-2025-01-09/tpch_queries_compare.png" /></p>
 <p>The following chart shows how much Comet currently accelerates each query from the benchmark in relative terms.</p>
-<p><img alt="" src="../../_images/tpch_queries_speedup_rel.png" /></p>
+<p><img alt="" src="_static/images/benchmark-results/0.5.0-SNAPSHOT-2025-01-09/tpch_queries_speedup_rel.png" /></p>
 <p>The following chart shows how much Comet currently accelerates each query from the benchmark in absolute terms.</p>
-<p><img alt="" src="../../_images/tpch_queries_speedup_abs.png" /></p>
+<p><img alt="" src="_static/images/benchmark-results/0.5.0-SNAPSHOT-2025-01-09/tpch_queries_speedup_abs.png" /></p>
 <p>The raw results of these benchmarks in JSON format is available here:</p>
 <ul class="simple">
-<li><p><a class="reference download internal" download="" href="../../_downloads/9e848a9cc1377847bac6c7c280c15083/spark-tpch.json"><span class="xref download myst">Spark</span></a></p></li>
-<li><p><a class="reference download internal" download="" href="../../_downloads/180af7f5ca0d6ea4521d87c6058b110f/comet-tpch.json"><span class="xref download myst">Comet</span></a></p></li>
+<li><p><a class="reference download internal" download="" href="../../_downloads/eb4015c41cfdc9c2fc70045fbf7f0c84/spark-tpch.json"><span class="xref download myst">Spark</span></a></p></li>
+<li><p><a class="reference download internal" download="" href="../../_downloads/a57370497e8278995e02e50496e3d7d8/comet-tpch.json"><span class="xref download myst">Comet</span></a></p></li>
 </ul>
 </section>
 
diff --git a/contributor-guide/benchmarking.html b/contributor-guide/benchmarking.html
index 75347efa..720c6994 100644
--- a/contributor-guide/benchmarking.html
+++ b/contributor-guide/benchmarking.html
@@ -295,6 +295,18 @@ under the License.
   <a class="reference internal nav-link" href="#running-benchmarks-against-apache-spark-with-apache-datafusion-comet-enabled">
    Running Benchmarks Against Apache Spark with Apache DataFusion Comet Enabled
   </a>
+  <ul class="nav section-nav flex-column">
+   <li class="toc-h3 nav-item toc-entry">
+    <a class="reference internal nav-link" href="#tpc-h">
+     TPC-H
+    </a>
+   </li>
+   <li class="toc-h3 nav-item toc-entry">
+    <a class="reference internal nav-link" href="#tpc-ds">
+     TPC-DS
+    </a>
+   </li>
+  </ul>
  </li>
  <li class="toc-h2 nav-item toc-entry">
   <a class="reference internal nav-link" href="#current-benchmark-results">
@@ -375,6 +387,8 @@ repository.</p>
 </section>
 <section id="running-benchmarks-against-apache-spark-with-apache-datafusion-comet-enabled">
 <h2>Running Benchmarks Against Apache Spark with Apache DataFusion Comet Enabled<a class="headerlink" href="#running-benchmarks-against-apache-spark-with-apache-datafusion-comet-enabled" title="Link to this heading">¶</a></h2>
+<section id="tpc-h">
+<h3>TPC-H<a class="headerlink" href="#tpc-h" title="Link to this heading">¶</a></h3>
 <div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nv">$SPARK_HOME</span>/bin/spark-submit<span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--master<span class="w"> </span><span class="nv">$SPARK_MASTER</span><span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--conf<span class="w"> </span>spark.driver.memory<span class="o">=</span>8G<span class="w"> </span><span class="se">\</span>
@@ -383,15 +397,19 @@ repository.</p>
 <span class="w">    </span>--conf<span class="w"> </span>spark.executor.cores<span class="o">=</span><span class="m">8</span><span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--conf<span class="w"> </span>spark.cores.max<span class="o">=</span><span class="m">8</span><span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--conf<span class="w"> </span>spark.memory.offHeap.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w">    </span>--conf<span class="w"> </span>spark.memory.offHeap.size<span class="o">=</span>32g<span class="w"> </span><span class="se">\</span>
+<span class="w">    </span>--conf<span class="w"> </span>spark.memory.offHeap.size<span class="o">=</span>16g<span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--jars<span class="w"> </span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--conf<span class="w"> </span>spark.driver.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--conf<span class="w"> </span>spark.executor.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--conf<span class="w"> </span>spark.plugins<span class="o">=</span>org.apache.spark.CometPlugin<span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--conf<span class="w"> </span>spark.comet.cast.allowIncompatible<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--conf<span class="w"> </span>spark.comet.exec.replaceSortMergeJoin<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w">    </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--conf<span class="w"> </span>spark.shuffle.manager<span class="o">=</span>org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager<span class="w"> </span><span class="se">\</span>
+<span class="w">    </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
+<span class="w">    </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.mode<span class="o">=</span>auto<span class="w"> </span><span class="se">\</span>
+<span class="w">    </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.enableFastEncoding<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
+<span class="w">    </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.fallbackToColumnar<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
+<span class="w">    </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.compression.codec<span class="o">=</span>lz4<span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>tpcbench.py<span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--benchmark<span class="w"> </span>tpch<span class="w"> </span><span class="se">\</span>
 <span class="w">    </span>--data<span class="w"> </span>/mnt/bigdata/tpch/sf100/<span class="w"> </span><span class="se">\</span>
@@ -400,6 +418,11 @@ repository.</p>
 </pre></div>
 </div>
 </section>
+<section id="tpc-ds">
+<h3>TPC-DS<a class="headerlink" href="#tpc-ds" title="Link to this heading">¶</a></h3>
+<p>For TPC-DS, use <code class="docutils literal notranslate"><span class="pre">spark.comet.exec.replaceSortMergeJoin=false</span></code>.</p>
+</section>
+</section>
 <section id="current-benchmark-results">
 <h2>Current Benchmark Results<a class="headerlink" href="#current-benchmark-results" title="Link to this heading">¶</a></h2>
 <ul class="simple">
diff --git a/searchindex.js b/searchindex.js
index f1df9fc8..0fe663d2 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"1. Install Comet": [[9, "install-comet"]], "2. Clone Spark and Apply Diff": [[9, "clone-spark-and-apply-diff"]], "3. Run Spark SQL Tests": [[9, "run-spark-sql-tests"]], "ANSI mode": [[11, "ansi-mode"]], "API Differences Between Spark Versions": [[0, "api-differences-between-spark-versions"]], "ASF Links": [[10, null]], "Adding Spark-side Tests for the New Expression": [[0, "adding-spark-side-tests-for-the-new-expression"]], "Adding a New Expression": [[0,  [...]
\ No newline at end of file
+Search.setIndex({"alltitles": {"1. Install Comet": [[9, "install-comet"]], "2. Clone Spark and Apply Diff": [[9, "clone-spark-and-apply-diff"]], "3. Run Spark SQL Tests": [[9, "run-spark-sql-tests"]], "ANSI mode": [[11, "ansi-mode"]], "API Differences Between Spark Versions": [[0, "api-differences-between-spark-versions"]], "ASF Links": [[10, null]], "Adding Spark-side Tests for the New Expression": [[0, "adding-spark-side-tests-for-the-new-expression"]], "Adding a New Expression": [[0,  [...]
\ No newline at end of file
diff --git a/user-guide/configs.html b/user-guide/configs.html
index 0df2ecab..3a7aa42e 100644
--- a/user-guide/configs.html
+++ b/user-guide/configs.html
@@ -430,131 +430,135 @@ under the License.
 <td><p>Whether to enable hashJoin by default.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.exec.localLimit.enabled</p></td>
+<tr class="row-odd"><td><p>spark.comet.exec.initCap.enabled</p></td>
+<td><p>Whether to enable initCap by default.</p></td>
+<td><p>false</p></td>
+</tr>
+<tr class="row-even"><td><p>spark.comet.exec.localLimit.enabled</p></td>
 <td><p>Whether to enable localLimit by default.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.exec.memoryPool</p></td>
+<tr class="row-odd"><td><p>spark.comet.exec.memoryPool</p></td>
 <td><p>The type of memory pool to be used for Comet native execution. Available memory pool types are ‘greedy’, ‘fair_spill’, ‘greedy_task_shared’, ‘fair_spill_task_shared’, ‘greedy_global’ and ‘fair_spill_global’, By default, this config is ‘greedy_task_shared’.</p></td>
 <td><p>greedy_task_shared</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.exec.project.enabled</p></td>
+<tr class="row-even"><td><p>spark.comet.exec.project.enabled</p></td>
 <td><p>Whether to enable project by default.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.exec.replaceSortMergeJoin</p></td>
+<tr class="row-odd"><td><p>spark.comet.exec.replaceSortMergeJoin</p></td>
 <td><p>Experimental feature to force Spark to replace SortMergeJoin with ShuffledHashJoin for improved performance. This feature is not stable yet. For more information, refer to the Comet Tuning Guide (https://datafusion.apache.org/comet/user-guide/tuning.html).</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.exec.shuffle.compression.codec</p></td>
+<tr class="row-even"><td><p>spark.comet.exec.shuffle.compression.codec</p></td>
 <td><p>The codec of Comet native shuffle used to compress shuffle data. lz4, zstd, and snappy are supported. Compression can be disabled by setting spark.shuffle.compress=false.</p></td>
 <td><p>lz4</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.exec.shuffle.compression.zstd.level</p></td>
+<tr class="row-odd"><td><p>spark.comet.exec.shuffle.compression.zstd.level</p></td>
 <td><p>The compression level to use when compressing shuffle files with zstd.</p></td>
 <td><p>1</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.exec.shuffle.enabled</p></td>
+<tr class="row-even"><td><p>spark.comet.exec.shuffle.enabled</p></td>
 <td><p>Whether to enable Comet native shuffle. Note that this requires setting ‘spark.shuffle.manager’ to ‘org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager’. ‘spark.shuffle.manager’ must be set before starting the Spark application and cannot be changed during the application.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.exec.sort.enabled</p></td>
+<tr class="row-odd"><td><p>spark.comet.exec.sort.enabled</p></td>
 <td><p>Whether to enable sort by default.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.exec.sortMergeJoin.enabled</p></td>
+<tr class="row-even"><td><p>spark.comet.exec.sortMergeJoin.enabled</p></td>
 <td><p>Whether to enable sortMergeJoin by default.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.exec.sortMergeJoinWithJoinFilter.enabled</p></td>
+<tr class="row-odd"><td><p>spark.comet.exec.sortMergeJoinWithJoinFilter.enabled</p></td>
 <td><p>Experimental support for Sort Merge Join with filter</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.exec.stddev.enabled</p></td>
+<tr class="row-even"><td><p>spark.comet.exec.stddev.enabled</p></td>
 <td><p>Whether to enable stddev by default. stddev is slower than Spark’s implementation.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.exec.takeOrderedAndProject.enabled</p></td>
+<tr class="row-odd"><td><p>spark.comet.exec.takeOrderedAndProject.enabled</p></td>
 <td><p>Whether to enable takeOrderedAndProject by default.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.exec.union.enabled</p></td>
+<tr class="row-even"><td><p>spark.comet.exec.union.enabled</p></td>
 <td><p>Whether to enable union by default.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.exec.window.enabled</p></td>
+<tr class="row-odd"><td><p>spark.comet.exec.window.enabled</p></td>
 <td><p>Whether to enable window by default.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.explain.native.enabled</p></td>
+<tr class="row-even"><td><p>spark.comet.explain.native.enabled</p></td>
 <td><p>When this setting is enabled, Comet will provide a tree representation of the native query plan before execution and again after execution, with metrics.</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.explain.verbose.enabled</p></td>
+<tr class="row-odd"><td><p>spark.comet.explain.verbose.enabled</p></td>
 <td><p>When this setting is enabled, Comet will provide a verbose tree representation of the extended information.</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.explainFallback.enabled</p></td>
+<tr class="row-even"><td><p>spark.comet.explainFallback.enabled</p></td>
 <td><p>When this setting is enabled, Comet will provide logging explaining the reason(s) why a query stage cannot be executed natively. Set this to false to reduce the amount of logging.</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.memory.overhead.factor</p></td>
+<tr class="row-odd"><td><p>spark.comet.memory.overhead.factor</p></td>
 <td><p>Fraction of executor memory to be allocated as additional non-heap memory per executor process for Comet.</p></td>
 <td><p>0.2</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.memory.overhead.min</p></td>
+<tr class="row-even"><td><p>spark.comet.memory.overhead.min</p></td>
 <td><p>Minimum amount of additional memory to be allocated per executor process for Comet, in MiB.</p></td>
 <td><p>402653184b</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.nativeLoadRequired</p></td>
+<tr class="row-odd"><td><p>spark.comet.nativeLoadRequired</p></td>
 <td><p>Whether to require Comet native library to load successfully when Comet is enabled. If not, Comet will silently fallback to Spark when it fails to load the native lib. Otherwise, an error will be thrown and the Spark job will be aborted.</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.parquet.enable.directBuffer</p></td>
+<tr class="row-even"><td><p>spark.comet.parquet.enable.directBuffer</p></td>
 <td><p>Whether to use Java direct byte buffer when reading Parquet.</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.parquet.read.io.adjust.readRange.skew</p></td>
+<tr class="row-odd"><td><p>spark.comet.parquet.read.io.adjust.readRange.skew</p></td>
 <td><p>In the parallel reader, if the read ranges submitted are skewed in sizes, this option will cause the reader to break up larger read ranges into smaller ranges to reduce the skew. This will result in a slightly larger number of connections opened to the file system but may give improved performance.</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.parquet.read.io.mergeRanges</p></td>
+<tr class="row-even"><td><p>spark.comet.parquet.read.io.mergeRanges</p></td>
 <td><p>When enabled the parallel reader will try to merge ranges of data that are separated by less than ‘comet.parquet.read.io.mergeRanges.delta’ bytes. Longer continuous reads are faster on cloud storage.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.parquet.read.io.mergeRanges.delta</p></td>
+<tr class="row-odd"><td><p>spark.comet.parquet.read.io.mergeRanges.delta</p></td>
 <td><p>The delta in bytes between consecutive read ranges below which the parallel reader will try to merge the ranges. The default is 8MB.</p></td>
 <td><p>8388608</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.parquet.read.parallel.io.enabled</p></td>
+<tr class="row-even"><td><p>spark.comet.parquet.read.parallel.io.enabled</p></td>
 <td><p>Whether to enable Comet’s parallel reader for Parquet files. The parallel reader reads ranges of consecutive data in a  file in parallel. It is faster for large files and row groups but uses more resources.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.parquet.read.parallel.io.thread-pool.size</p></td>
+<tr class="row-odd"><td><p>spark.comet.parquet.read.parallel.io.thread-pool.size</p></td>
 <td><p>The maximum number of parallel threads the parallel reader will use in a single executor. For executors configured with a smaller number of cores, use a smaller number.</p></td>
 <td><p>16</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.regexp.allowIncompatible</p></td>
+<tr class="row-even"><td><p>spark.comet.regexp.allowIncompatible</p></td>
 <td><p>Comet is not currently fully compatible with Spark for all regular expressions. Set this config to true to allow them anyway using Rust’s regular expression engine. See compatibility guide for more information.</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.scan.enabled</p></td>
+<tr class="row-odd"><td><p>spark.comet.scan.enabled</p></td>
 <td><p>Whether to enable native scans. When this is turned on, Spark will use Comet to read supported data sources (currently only Parquet is supported natively). Note that to enable native vectorized execution, both this config and ‘spark.comet.exec.enabled’ need to be enabled.</p></td>
 <td><p>true</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.scan.preFetch.enabled</p></td>
+<tr class="row-even"><td><p>spark.comet.scan.preFetch.enabled</p></td>
 <td><p>Whether to enable pre-fetching feature of CometScan.</p></td>
 <td><p>false</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.scan.preFetch.threadNum</p></td>
+<tr class="row-odd"><td><p>spark.comet.scan.preFetch.threadNum</p></td>
 <td><p>The number of threads running pre-fetching for CometScan. Effective if spark.comet.scan.preFetch.enabled is enabled. Note that more pre-fetching threads means more memory requirement to store pre-fetched row groups.</p></td>
 <td><p>2</p></td>
 </tr>
-<tr class="row-odd"><td><p>spark.comet.shuffle.preferDictionary.ratio</p></td>
+<tr class="row-even"><td><p>spark.comet.shuffle.preferDictionary.ratio</p></td>
 <td><p>The ratio of total values to distinct values in a string column to decide whether to prefer dictionary encoding when shuffling the column. If the ratio is higher than this config, dictionary encoding will be used on shuffling string column. This config is effective if it is higher than 1.0. Note that this config is only used when <code class="docutils literal notranslate"><span class="pre">spark.comet.exec.shuffle.mode</span></code> is <code class="docutils literal notranslate"><s [...]
 <td><p>10.0</p></td>
 </tr>
-<tr class="row-even"><td><p>spark.comet.sparkToColumnar.supportedOperatorList</p></td>
+<tr class="row-odd"><td><p>spark.comet.sparkToColumnar.supportedOperatorList</p></td>
 <td><p>A comma-separated list of operators that will be converted to Arrow columnar format when ‘spark.comet.sparkToColumnar.enabled’ is true</p></td>
 <td><p>Range,InMemoryTableScan</p></td>
 </tr>


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org