You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2025/03/19 17:05:08 UTC
(datafusion-comet) branch asf-site updated: Publish built docs triggered by 56a52137d8c59f7e68a3c7d1b05ffe6867f78f8e
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 07c66929e Publish built docs triggered by 56a52137d8c59f7e68a3c7d1b05ffe6867f78f8e
07c66929e is described below
commit 07c66929e93f5d3dc9bf10baf870d338d267c5c5
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Wed Mar 19 17:04:57 2025 +0000
Publish built docs triggered by 56a52137d8c59f7e68a3c7d1b05ffe6867f78f8e
---
.../spark-tpcds.json | 347 ++++++++++
.../comet-tpcds.json | 744 ---------------------
.../comet-tpch.json | 209 ------
.../comet-tpch.json | 134 ++++
.../spark-tpcds.json | 722 --------------------
.../spark-tpch.json | 185 -----
.../spark-tpch.json | 116 ++++
.../comet-tpcds.json | 361 ++++++++++
_images/tpcds_allqueries.png | Bin 27773 -> 31696 bytes
_images/tpcds_queries_compare.png | Bin 42959 -> 44747 bytes
_images/tpcds_queries_speedup_abs.png | Bin 66470 -> 67857 bytes
_images/tpcds_queries_speedup_rel.png | Bin 91701 -> 92770 bytes
_images/tpch_allqueries.png | Bin 28138 -> 31939 bytes
_images/tpch_queries_compare.png | Bin 33059 -> 34385 bytes
_images/tpch_queries_speedup_abs.png | Bin 38156 -> 38064 bytes
_images/tpch_queries_speedup_rel.png | Bin 48224 -> 47901 bytes
.../benchmark-results/tpc-ds.md.txt | 95 ++-
.../benchmark-results/tpc-h.md.txt | 87 ++-
.../benchmark-results/0.2.0/tpcds_allqueries.png | Bin 23827 -> 0 bytes
.../0.2.0/tpcds_queries_compare.png | Bin 38305 -> 0 bytes
.../0.2.0/tpcds_queries_speedup_abs.png | Bin 62949 -> 0 bytes
.../benchmark-results/0.2.0/tpch_allqueries.png | Bin 26062 -> 0 bytes
.../0.2.0/tpch_queries_compare.png | Bin 29207 -> 0 bytes
.../0.2.0/tpch_queries_speedup_rel.png | Bin 43233 -> 0 bytes
.../benchmark-results/0.3.0/tpcds_allqueries.png | Bin 30139 -> 0 bytes
.../0.3.0/tpcds_queries_compare.png | Bin 46449 -> 0 bytes
.../0.3.0/tpcds_queries_speedup_abs.png | Bin 67006 -> 0 bytes
.../0.3.0/tpcds_queries_speedup_rel.png | Bin 96045 -> 0 bytes
.../benchmark-results/0.3.0/tpch_allqueries.png | Bin 29289 -> 0 bytes
.../0.3.0/tpch_queries_compare.png | Bin 33052 -> 0 bytes
.../0.3.0/tpch_queries_speedup_abs.png | Bin 37680 -> 0 bytes
.../0.3.0/tpch_queries_speedup_rel.png | Bin 44133 -> 0 bytes
.../benchmark-results/0.4.0/tpcds_allqueries.png | Bin 28503 -> 0 bytes
.../0.4.0/tpcds_queries_compare.png | Bin 43094 -> 0 bytes
.../0.4.0/tpcds_queries_speedup_abs.png | Bin 64166 -> 0 bytes
.../0.4.0/tpcds_queries_speedup_rel.png | Bin 90369 -> 0 bytes
.../benchmark-results/0.4.0/tpch_allqueries.png | Bin 29251 -> 0 bytes
.../0.4.0/tpch_queries_compare.png | Bin 34357 -> 0 bytes
.../0.4.0/tpch_queries_speedup_abs.png | Bin 37579 -> 0 bytes
.../0.4.0/tpch_queries_speedup_rel.png | Bin 48051 -> 0 bytes
.../benchmark-results/0.7.0/tpcds_allqueries.png | Bin 0 -> 31696 bytes
.../0.7.0/tpcds_queries_compare.png | Bin 0 -> 44747 bytes
.../0.7.0/tpcds_queries_speedup_abs.png | Bin 0 -> 67857 bytes
.../0.7.0/tpcds_queries_speedup_rel.png | Bin 0 -> 92770 bytes
.../benchmark-results/0.7.0/tpch_allqueries.png | Bin 0 -> 31939 bytes
.../0.7.0/tpch_queries_compare.png | Bin 0 -> 34385 bytes
.../0.7.0/tpch_queries_speedup_abs.png | Bin 0 -> 38064 bytes
.../0.7.0/tpch_queries_speedup_rel.png | Bin 0 -> 47901 bytes
contributor-guide/benchmark-results/tpc-ds.html | 87 ++-
contributor-guide/benchmark-results/tpc-h.html | 80 ++-
searchindex.js | 2 +-
51 files changed, 1186 insertions(+), 1983 deletions(-)
diff --git a/_downloads/4cc7a10f09550a16fb1b753e4fd5d0e2/spark-tpcds.json b/_downloads/4cc7a10f09550a16fb1b753e4fd5d0e2/spark-tpcds.json
new file mode 100644
index 000000000..5fbd812e5
--- /dev/null
+++ b/_downloads/4cc7a10f09550a16fb1b753e4fd5d0e2/spark-tpcds.json
@@ -0,0 +1,347 @@
+{
+ "engine": "datafusion-comet",
+ "benchmark": "tpcds",
+ "data_path": "/mnt/bigdata/tpcds/sf100/",
+ "query_path": "/tpcds",
+ "spark_conf": {
+ "spark.app.submitTime": "1742328355539",
+ "spark.kubernetes.driver.pod.name": "tpcbench-py-00f9af95aadcd249-driver",
+ "spark.kubernetes.executor.container.image": "apache/spark:3.5.3",
+ "spark.kubernetes.executor.podNamePrefix": "spark-benchmark-derived-from-tpcds-cf3eb995aadcdd4e",
+ "spark.kubernetes.memoryOverheadFactor": "0.4",
+ "spark.kubernetes.authenticate.driver.serviceAccountName": "spark",
+ "spark.kubernetes.driver.volumes.hostPath.data.mount.path": "/mnt/bigdata",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName": "benchmark-results-pvc",
+ "spark.memory.offHeap.enabled": "true",
+ "spark.memory.offHeap.size": "32g",
+ "spark.driver.bindAddress": "10.42.0.31",
+ "spark.app.startTime": "1742328355796",
+ "spark.master": "k8s://https://kube1:6443",
+ "spark.driver.host": "tpcbench-py-00f9af95aadcd249-driver-svc.default.svc",
+ "spark.kubernetes.driver.container.image": "woody:5000/spark-tpcbench:spark",
+ "spark.serializer.objectStreamReset": "100",
+ "spark.executor.cores": "16",
+ "spark.submit.deployMode": "client",
+ "spark.kubernetes.driver.volumes.hostPath.data.options.path": "/mnt/bigdata",
+ "spark.eventLog.enabled": "false",
+ "spark.kubernetes.container.image.pullPolicy": "Always",
+ "spark.kubernetes.submitInDriver": "true",
+ "spark.executorEnv.SPARK_DRIVER_POD_IP": "10.42.0.31",
+ "spark.sql.warehouse.dir": "file:/opt/spark/work-dir/spark-warehouse",
+ "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
+ "spark.driver.blockManager.port": "7079",
+ "spark.app.id": "spark-6141725ef4a44a529bccc0cea1eb1ed4",
+ "spark.executor.id": "driver",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly": "false",
+ "spark.driver.memory": "8G",
+ "spark.driver.port": "7078",
+ "spark.executor.memory": "32g",
+ "spark.kubernetes.driver.volumes.hostPath.data.options.readOnly": "false",
+ "spark.kubernetes.resource.type": "python",
+ "spark.kubernetes.executor.volumes.hostPath.data.mount.path": "/mnt/bigdata",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path": "/mnt/benchmark-results",
+ "spark.rdd.compress": "True",
+ "spark.kubernetes.executor.volumes.hostPath.data.options.path": "/mnt/bigdata",
+ "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
+ "spark.executor.instances": "1",
+ "spark.submit.pyFiles": "",
+ "spark.app.name": "spark benchmark derived from tpcds"
+ },
+ "1": [
+ 4.355947971343994
+ ],
+ "2": [
+ 3.489915609359741
+ ],
+ "3": [
+ 1.488908290863037
+ ],
+ "4": [
+ 57.785072565078735
+ ],
+ "5": [
+ 13.763205528259277
+ ],
+ "6": [
+ 2.2846295833587646
+ ],
+ "7": [
+ 3.3782906532287598
+ ],
+ "8": [
+ 2.400043487548828
+ ],
+ "9": [
+ 5.938290357589722
+ ],
+ "10": [
+ 2.449601888656616
+ ],
+ "11": [
+ 16.143503427505493
+ ],
+ "12": [
+ 1.4717519283294678
+ ],
+ "13": [
+ 4.343080997467041
+ ],
+ "14": [
+ 44.92432475090027
+ ],
+ "15": [
+ 3.5471317768096924
+ ],
+ "16": [
+ 13.545910120010376
+ ],
+ "17": [
+ 4.942067861557007
+ ],
+ "18": [
+ 3.94262433052063
+ ],
+ "19": [
+ 3.962956666946411
+ ],
+ "20": [
+ 1.3886706829071045
+ ],
+ "21": [
+ 1.7535080909729004
+ ],
+ "22": [
+ 20.089434146881104
+ ],
+ "23": [
+ 103.30900120735168
+ ],
+ "24": [
+ 16.53474521636963
+ ],
+ "25": [
+ 4.266413927078247
+ ],
+ "26": [
+ 2.4301295280456543
+ ],
+ "27": [
+ 2.811310291290283
+ ],
+ "28": [
+ 8.736858129501343
+ ],
+ "29": [
+ 4.963813066482544
+ ],
+ "30": [
+ 2.074880838394165
+ ],
+ "31": [
+ 6.412190675735474
+ ],
+ "32": [
+ 2.206458330154419
+ ],
+ "33": [
+ 3.412022352218628
+ ],
+ "34": [
+ 1.658189058303833
+ ],
+ "35": [
+ 3.984631061553955
+ ],
+ "36": [
+ 2.2529945373535156
+ ],
+ "37": [
+ 4.328810930252075
+ ],
+ "38": [
+ 7.559122323989868
+ ],
+ "39": [
+ 8.816664934158325
+ ],
+ "40": [
+ 8.136451244354248
+ ],
+ "41": [
+ 0.25649333000183105
+ ],
+ "42": [
+ 1.4119703769683838
+ ],
+ "43": [
+ 1.3050105571746826
+ ],
+ "44": [
+ 1.144343614578247
+ ],
+ "45": [
+ 2.252218008041382
+ ],
+ "46": [
+ 3.073315143585205
+ ],
+ "47": [
+ 5.461176633834839
+ ],
+ "48": [
+ 8.714987754821777
+ ],
+ "49": [
+ 7.581437826156616
+ ],
+ "50": [
+ 12.17387843132019
+ ],
+ "51": [
+ 14.94502067565918
+ ],
+ "52": [
+ 1.7854347229003906
+ ],
+ "53": [
+ 1.8280425071716309
+ ],
+ "54": [
+ 3.6002089977264404
+ ],
+ "55": [
+ 1.6423735618591309
+ ],
+ "56": [
+ 3.359402894973755
+ ],
+ "57": [
+ 3.3884055614471436
+ ],
+ "58": [
+ 4.283772230148315
+ ],
+ "59": [
+ 3.650860548019409
+ ],
+ "60": [
+ 3.8869588375091553
+ ],
+ "61": [
+ 3.69657039642334
+ ],
+ "62": [
+ 0.879765510559082
+ ],
+ "63": [
+ 1.6274287700653076
+ ],
+ "64": [
+ 25.166638374328613
+ ],
+ "65": [
+ 12.127745628356934
+ ],
+ "66": [
+ 3.445380687713623
+ ],
+ "67": [
+ 66.29370522499084
+ ],
+ "68": [
+ 2.7156710624694824
+ ],
+ "69": [
+ 2.5902113914489746
+ ],
+ "70": [
+ 2.810850143432617
+ ],
+ "71": [
+ 3.2944750785827637
+ ],
+ "72": [
+ 89.93545317649841
+ ],
+ "73": [
+ 1.326775074005127
+ ],
+ "74": [
+ 23.207229137420654
+ ],
+ "75": [
+ 13.160036325454712
+ ],
+ "76": [
+ 3.329620361328125
+ ],
+ "77": [
+ 3.584099769592285
+ ],
+ "78": [
+ 33.91078543663025
+ ],
+ "79": [
+ 2.310992956161499
+ ],
+ "80": [
+ 26.44196081161499
+ ],
+ "81": [
+ 2.294264554977417
+ ],
+ "82": [
+ 2.545654296875
+ ],
+ "83": [
+ 1.7201306819915771
+ ],
+ "84": [
+ 1.502422571182251
+ ],
+ "85": [
+ 5.089749336242676
+ ],
+ "86": [
+ 1.043938398361206
+ ],
+ "87": [
+ 7.822947263717651
+ ],
+ "88": [
+ 5.241043567657471
+ ],
+ "89": [
+ 2.0387654304504395
+ ],
+ "90": [
+ 0.8709492683410645
+ ],
+ "91": [
+ 1.0122673511505127
+ ],
+ "92": [
+ 1.4276893138885498
+ ],
+ "93": [
+ 23.040839910507202
+ ],
+ "94": [
+ 7.798746585845947
+ ],
+ "95": [
+ 28.01142120361328
+ ],
+ "96": [
+ 0.9283573627471924
+ ],
+ "97": [
+ 11.431622982025146
+ ],
+ "98": [
+ 2.4064207077026367
+ ],
+ "99": [
+ 1.321455478668213
+ ]
+}
\ No newline at end of file
diff --git a/_downloads/a2bf8d2de6f449ff2ab1739c3a17f18e/comet-tpcds.json b/_downloads/a2bf8d2de6f449ff2ab1739c3a17f18e/comet-tpcds.json
deleted file mode 100644
index 8d249e3c7..000000000
--- a/_downloads/a2bf8d2de6f449ff2ab1739c3a17f18e/comet-tpcds.json
+++ /dev/null
@@ -1,744 +0,0 @@
-{
- "engine": "datafusion-comet",
- "benchmark": "tpcds",
- "data_path": "/mnt/bigdata/tpcds/sf100/",
- "query_path": "../../tpcds/",
- "spark_conf": {
- "spark.executor.memoryOverhead": "4914M",
- "spark.eventLog.enabled": "true",
- "spark.executor.extraClassPath": "/home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.comet.cast.allowIncompatible": "true",
- "spark.app.submitTime": "1736811668530",
- "spark.comet.explainFallback.enabled": "false",
- "spark.cores.max": "16",
- "spark.comet.exec.shuffle.enabled": "true",
- "spark.app.startTime": "1736811668800",
- "spark.memory.offHeap.enabled": "true",
- "spark.driver.port": "42815",
- "spark.executor.instances": "2",
- "spark.executor.memory": "16G",
- "spark.comet.exec.shuffle.fallbackToColumnar": "true",
- "spark.serializer.objectStreamReset": "100",
- "spark.driver.host": "10.0.0.118",
- "spark.submit.deployMode": "client",
- "spark.executor.cores": "8",
- "spark.driver.extraClassPath": "/home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.app.id": "app-20250113164109-0060",
- "spark.comet.exec.replaceSortMergeJoin": "false",
- "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
- "spark.driver.maxResultSize": "2G",
- "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
- "spark.sql.warehouse.dir": "file:/home/andy/git/personal/research/benchmarks/spark-standalone/spark-warehouse",
- "spark.executor.id": "driver",
- "spark.master": "spark://woody:7077",
- "spark.comet.exec.shuffle.mode": "auto",
- "spark.sql.extensions": "org.apache.comet.CometSparkSessionExtensions",
- "spark.driver.memory": "8G",
- "spark.plugins": "org.apache.spark.CometPlugin",
- "spark.repl.local.jars": "file:///home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.memory.offHeap.size": "24g",
- "spark.rdd.compress": "True",
- "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
- "spark.comet.enabled": "true",
- "spark.submit.pyFiles": "",
- "spark.app.name": "comet benchmark derived from tpcds",
- "spark.comet.exec.sortMergeJoinWithJoinFilter.enabled": "false",
- "spark.app.initial.jar.urls": "spark://10.0.0.118:42815/jars/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.comet.exec.shuffle.compression.codec": "lz4",
- "spark.jars": "file:///home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.comet.explain.native.enabled": "false"
- },
- "1": [
- 4.536968469619751,
- 1.3565826416015625,
- 1.338052749633789,
- 1.4378666877746582,
- 1.3548147678375244
- ],
- "2": [
- 3.3205230236053467,
- 2.462737798690796,
- 2.488901376724243,
- 2.5463438034057617,
- 2.527088165283203
- ],
- "3": [
- 1.6579983234405518,
- 1.47391939163208,
- 1.4881572723388672,
- 1.484151840209961,
- 1.5221748352050781
- ],
- "4": [
- 34.67596936225891,
- 33.768837451934814,
- 34.09213376045227,
- 34.43401646614075,
- 34.43135046958923
- ],
- "5": [
- 6.600037097930908,
- 6.366944074630737,
- 6.282367467880249,
- 6.359925985336304,
- 6.503366231918335
- ],
- "6": [
- 1.7636768817901611,
- 1.6569476127624512,
- 1.564842939376831,
- 1.6130192279815674,
- 1.562074899673462
- ],
- "7": [
- 3.080796480178833,
- 3.0056214332580566,
- 3.1245322227478027,
- 3.1279425621032715,
- 3.1556527614593506
- ],
- "8": [
- 1.871955394744873,
- 1.6913323402404785,
- 1.6697134971618652,
- 1.7257518768310547,
- 1.782395839691162
- ],
- "9": [
- 6.713606357574463,
- 6.480487108230591,
- 6.67946982383728,
- 6.535262823104858,
- 6.520689249038696
- ],
- "10": [
- 2.2367494106292725,
- 2.069857120513916,
- 2.1344666481018066,
- 2.0363876819610596,
- 2.0629072189331055
- ],
- "11": [
- 13.441156148910522,
- 13.661643505096436,
- 13.749130487442017,
- 13.663432121276855,
- 13.914729118347168
- ],
- "12": [
- 1.4952607154846191,
- 1.327760934829712,
- 1.3590035438537598,
- 1.3424949645996094,
- 1.3338284492492676
- ],
- "13": [
- 4.1215620040893555,
- 4.143518447875977,
- 4.115273952484131,
- 4.1204657554626465,
- 4.203814268112183
- ],
- "14": [
- 27.271831274032593,
- 27.740113973617554,
- 27.78661036491394,
- 28.459989070892334,
- 28.01462411880493
- ],
- "15": [
- 3.972994327545166,
- 4.010797739028931,
- 4.036948204040527,
- 4.097492933273315,
- 4.126002788543701
- ],
- "16": [
- 9.47628664970398,
- 9.299378871917725,
- 9.3838791847229,
- 9.37733244895935,
- 9.319334745407104
- ],
- "17": [
- 3.5423672199249268,
- 3.4606869220733643,
- 3.5566649436950684,
- 3.574188709259033,
- 3.5854296684265137
- ],
- "18": [
- 3.558619499206543,
- 3.7194430828094482,
- 3.720961332321167,
- 3.7957425117492676,
- 3.7437338829040527
- ],
- "19": [
- 4.280632495880127,
- 4.2238616943359375,
- 4.188013792037964,
- 4.338703155517578,
- 4.348641633987427
- ],
- "20": [
- 2.1416103839874268,
- 2.134965419769287,
- 2.132283926010132,
- 2.2123031616210938,
- 2.1274831295013428
- ],
- "21": [
- 1.6397242546081543,
- 1.6214661598205566,
- 1.5790197849273682,
- 1.5165207386016846,
- 1.5434236526489258
- ],
- "22": [
- 7.052053689956665,
- 7.3520801067352295,
- 7.85315465927124,
- 7.75153112411499,
- 7.7829179763793945
- ],
- "23": [
- 43.88178491592407,
- 44.70656991004944,
- 44.685741901397705,
- 45.05996298789978,
- 44.751373291015625
- ],
- "24": [
- 16.058145999908447,
- 16.151873111724854,
- 16.42806100845337,
- 16.49088191986084,
- 16.125349283218384
- ],
- "25": [
- 3.971809148788452,
- 4.022783279418945,
- 3.9840853214263916,
- 4.052400588989258,
- 4.11824893951416
- ],
- "26": [
- 1.5638508796691895,
- 1.5986883640289307,
- 1.596513271331787,
- 1.7379155158996582,
- 1.6375956535339355
- ],
- "27": [
- 2.977374315261841,
- 3.2149150371551514,
- 3.2501866817474365,
- 3.2618322372436523,
- 3.1989095211029053
- ],
- "28": [
- 6.967320203781128,
- 6.830939769744873,
- 6.937035799026489,
- 7.009445428848267,
- 7.003203392028809
- ],
- "29": [
- 5.793797969818115,
- 5.6882500648498535,
- 5.628785848617554,
- 5.72254753112793,
- 5.698575019836426
- ],
- "30": [
- 1.488220453262329,
- 1.3394947052001953,
- 1.3159022331237793,
- 1.3776261806488037,
- 1.3321137428283691
- ],
- "31": [
- 9.204715251922607,
- 8.69102168083191,
- 8.614688873291016,
- 8.787571668624878,
- 8.771551370620728
- ],
- "32": [
- 2.203498125076294,
- 2.1785271167755127,
- 2.0557124614715576,
- 2.211980104446411,
- 2.1879453659057617
- ],
- "33": [
- 3.9984161853790283,
- 4.1804728507995605,
- 4.131994247436523,
- 4.213898658752441,
- 4.188544034957886
- ],
- "34": [
- 1.875131368637085,
- 1.9190173149108887,
- 1.9300975799560547,
- 1.9599437713623047,
- 1.869981288909912
- ],
- "35": [
- 3.308222532272339,
- 3.061551332473755,
- 3.043506145477295,
- 3.083115577697754,
- 3.160632371902466
- ],
- "36": [
- 3.769775629043579,
- 3.782244920730591,
- 3.828948736190796,
- 3.8956804275512695,
- 3.8769822120666504
- ],
- "37": [
- 2.8596391677856445,
- 2.8715765476226807,
- 2.8092164993286133,
- 2.891303777694702,
- 2.855314254760742
- ],
- "38": [
- 3.4315402507781982,
- 3.4382240772247314,
- 3.4430289268493652,
- 3.531895637512207,
- 3.455247163772583
- ],
- "39": [
- 19.4264817237854,
- 20.363921403884888,
- 19.874735593795776,
- 19.83957862854004,
- 19.403523683547974
- ],
- "40": [
- 5.129716634750366,
- 5.0438196659088135,
- 5.126132249832153,
- 5.1094677448272705,
- 5.055907487869263
- ],
- "41": [
- 0.14982271194458008,
- 0.1292414665222168,
- 0.16390562057495117,
- 0.134552001953125,
- 0.12277865409851074
- ],
- "42": [
- 1.2786576747894287,
- 1.2783203125,
- 1.3130738735198975,
- 1.3072741031646729,
- 1.3132829666137695
- ],
- "43": [
- 1.6863470077514648,
- 1.6458985805511475,
- 1.7347028255462646,
- 1.765425682067871,
- 1.7646584510803223
- ],
- "44": [
- 2.0259382724761963,
- 2.0535051822662354,
- 1.9940221309661865,
- 2.0074307918548584,
- 1.9784820079803467
- ],
- "45": [
- 2.4755594730377197,
- 2.326653003692627,
- 2.3411612510681152,
- 2.347200632095337,
- 2.477112054824829
- ],
- "46": [
- 3.9604787826538086,
- 3.8620941638946533,
- 3.7819931507110596,
- 3.939593553543091,
- 3.838043451309204
- ],
- "47": [
- 7.505825996398926,
- 7.342543125152588,
- 7.576608657836914,
- 7.725332736968994,
- 7.735831260681152
- ],
- "48": [
- 2.8078317642211914,
- 2.809493064880371,
- 2.826028347015381,
- 2.9383201599121094,
- 2.9243483543395996
- ],
- "49": [
- 7.083904266357422,
- 7.132755279541016,
- 6.944513320922852,
- 6.931026935577393,
- 6.920399188995361
- ],
- "50": [
- 6.608541250228882,
- 7.139689922332764,
- 6.71970796585083,
- 6.847019195556641,
- 6.901025772094727
- ],
- "51": [
- 7.687297344207764,
- 7.605288982391357,
- 7.882007360458374,
- 7.688341856002808,
- 7.858721017837524
- ],
- "52": [
- 1.515904188156128,
- 1.5196130275726318,
- 1.5246155261993408,
- 1.539452314376831,
- 1.529005765914917
- ],
- "53": [
- 1.6835079193115234,
- 1.7078402042388916,
- 1.7528693675994873,
- 1.754622459411621,
- 1.801300287246704
- ],
- "54": [
- 4.626687288284302,
- 4.8692405223846436,
- 4.670963764190674,
- 4.76296854019165,
- 4.630118370056152
- ],
- "55": [
- 1.5379326343536377,
- 1.559366226196289,
- 1.585902452468872,
- 1.5425207614898682,
- 1.5508875846862793
- ],
- "56": [
- 3.372633457183838,
- 3.4592878818511963,
- 3.4758856296539307,
- 3.563056707382202,
- 3.5439987182617188
- ],
- "57": [
- 3.630242109298706,
- 3.5384538173675537,
- 3.5876047611236572,
- 3.6837234497070312,
- 3.6626791954040527
- ],
- "58": [
- 6.306454181671143,
- 6.243661642074585,
- 6.297546148300171,
- 6.354742050170898,
- 6.31113076210022
- ],
- "59": [
- 3.9262125492095947,
- 3.9229207038879395,
- 3.985262632369995,
- 4.089562892913818,
- 4.107280969619751
- ],
- "60": [
- 3.506413698196411,
- 3.5923097133636475,
- 3.5940756797790527,
- 3.6000287532806396,
- 3.6011013984680176
- ],
- "61": [
- 7.301086664199829,
- 7.541475057601929,
- 7.927215576171875,
- 7.743303298950195,
- 7.7511231899261475
- ],
- "62": [
- 1.0287232398986816,
- 1.033036708831787,
- 1.0503861904144287,
- 1.053584337234497,
- 1.036078929901123
- ],
- "63": [
- 1.6723690032958984,
- 1.7417714595794678,
- 1.97025728225708,
- 1.807016372680664,
- 1.8281068801879883
- ],
- "64": [
- 15.89651346206665,
- 15.665365934371948,
- 16.01818823814392,
- 15.994118690490723,
- 16.00064516067505
- ],
- "65": [
- 7.785307884216309,
- 7.80276894569397,
- 7.865602016448975,
- 7.857702016830444,
- 7.766430616378784
- ],
- "66": [
- 3.1051344871520996,
- 3.1696081161499023,
- 3.300004243850708,
- 3.5985312461853027,
- 3.43051815032959
- ],
- "67": [
- 30.156902551651,
- 30.163670778274536,
- 30.0355327129364,
- 30.676511764526367,
- 30.507113456726074
- ],
- "68": [
- 4.870973825454712,
- 4.989592552185059,
- 4.550822973251343,
- 4.479772329330444,
- 5.1386706829071045
- ],
- "69": [
- 2.1566920280456543,
- 2.2332279682159424,
- 2.0824127197265625,
- 2.211796998977661,
- 2.105759620666504
- ],
- "70": [
- 3.654249668121338,
- 3.706080198287964,
- 3.664548873901367,
- 3.912018299102783,
- 3.7903084754943848
- ],
- "71": [
- 3.5541186332702637,
- 3.5453219413757324,
- 3.483576536178589,
- 3.603425979614258,
- 3.5967133045196533
- ],
- "72": [
- 85.7537453174591,
- 84.34932899475098,
- 85.4409408569336,
- 82.9489848613739,
- 86.19663500785828
- ],
- "73": [
- 1.5869219303131104,
- 1.5673043727874756,
- 1.5760784149169922,
- 1.6549866199493408,
- 1.6651747226715088
- ],
- "74": [
- 9.591845989227295,
- 9.609638690948486,
- 9.660242319107056,
- 9.578797101974487,
- 9.830672264099121
- ],
- "75": [
- 12.06259274482727,
- 12.131521224975586,
- 12.339975833892822,
- 12.526771783828735,
- 12.704570055007935
- ],
- "76": [
- 3.1800994873046875,
- 3.1969289779663086,
- 3.2074594497680664,
- 3.249382495880127,
- 3.2397830486297607
- ],
- "77": [
- 3.9328701496124268,
- 3.8905186653137207,
- 4.043585777282715,
- 4.000932216644287,
- 4.100231409072876
- ],
- "78": [
- 19.32929301261902,
- 18.75995111465454,
- 19.079716444015503,
- 18.925426721572876,
- 19.023631811141968
- ],
- "79": [
- 2.897693157196045,
- 3.135042667388916,
- 3.0352957248687744,
- 3.0536019802093506,
- 3.0584769248962402
- ],
- "80": [
- 15.195313692092896,
- 15.409253120422363,
- 15.783752918243408,
- 15.655308961868286,
- 15.55367660522461
- ],
- "81": [
- 1.6697850227355957,
- 1.6849262714385986,
- 1.7256522178649902,
- 1.6104223728179932,
- 1.5729634761810303
- ],
- "82": [
- 4.702307462692261,
- 4.762118101119995,
- 4.631009101867676,
- 4.632325172424316,
- 4.699774742126465
- ],
- "83": [
- 0.9655017852783203,
- 0.964923620223999,
- 1.0950005054473877,
- 0.9366519451141357,
- 0.9813294410705566
- ],
- "84": [
- 0.8160817623138428,
- 0.7901313304901123,
- 0.9309515953063965,
- 0.7813336849212646,
- 0.8049192428588867
- ],
- "85": [
- 2.1436941623687744,
- 2.0501821041107178,
- 2.3379878997802734,
- 2.2173709869384766,
- 2.0497560501098633
- ],
- "86": [
- 0.8833699226379395,
- 0.8855693340301514,
- 1.003610610961914,
- 0.9266695976257324,
- 0.8708605766296387
- ],
- "87": [
- 3.5058393478393555,
- 3.5163745880126953,
- 3.8271989822387695,
- 3.621147632598877,
- 3.5709948539733887
- ],
- "88": [
- 6.482669115066528,
- 6.477633714675903,
- 6.942981958389282,
- 6.804642677307129,
- 6.639838457107544
- ],
- "89": [
- 2.1237101554870605,
- 2.126112461090088,
- 2.1807539463043213,
- 2.172977924346924,
- 2.20328426361084
- ],
- "90": [
- 0.5848908424377441,
- 0.5502984523773193,
- 0.6235551834106445,
- 0.569770336151123,
- 0.5719239711761475
- ],
- "91": [
- 0.6266171932220459,
- 0.5699453353881836,
- 0.6588234901428223,
- 0.6302447319030762,
- 0.6418545246124268
- ],
- "92": [
- 0.8711974620819092,
- 0.8485770225524902,
- 0.8334972858428955,
- 0.8226969242095947,
- 0.8113462924957275
- ],
- "93": [
- 9.335367202758789,
- 9.483666181564331,
- 9.641432046890259,
- 9.4573495388031,
- 9.48092794418335
- ],
- "94": [
- 5.219858884811401,
- 5.30799126625061,
- 5.2076966762542725,
- 5.314690589904785,
- 5.152010202407837
- ],
- "95": [
- 20.359986066818237,
- 19.688702821731567,
- 20.081215381622314,
- 19.575989961624146,
- 21.01106309890747
- ],
- "96": [
- 0.8545236587524414,
- 0.8409044742584229,
- 0.8572230339050293,
- 0.8502027988433838,
- 0.9349768161773682
- ],
- "97": [
- 5.144511938095093,
- 5.298552751541138,
- 5.229865789413452,
- 5.0392138957977295,
- 5.193090915679932
- ],
- "98": [
- 4.391782283782959,
- 4.436994552612305,
- 4.410702705383301,
- 4.459229946136475,
- 4.5571510791778564
- ],
- "99": [
- 1.6422052383422852,
- 1.5400826930999756,
- 1.5657870769500732,
- 1.5555531978607178,
- 1.582702398300171
- ]
-}
\ No newline at end of file
diff --git a/_downloads/a57370497e8278995e02e50496e3d7d8/comet-tpch.json b/_downloads/a57370497e8278995e02e50496e3d7d8/comet-tpch.json
deleted file mode 100644
index b1b3ae127..000000000
--- a/_downloads/a57370497e8278995e02e50496e3d7d8/comet-tpch.json
+++ /dev/null
@@ -1,209 +0,0 @@
-{
- "engine": "datafusion-comet",
- "benchmark": "tpch",
- "data_path": "/mnt/bigdata/tpch/sf100/",
- "query_path": "/home/andy/git/apache/datafusion-benchmarks/tpch/queries",
- "spark_conf": {
- "spark.comet.explain.native.enabled": "false",
- "spark.eventLog.enabled": "true",
- "spark.executor.extraClassPath": "/home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.comet.explainFallback.enabled": "false",
- "spark.comet.exec.replaceSortMergeJoin": "true",
- "spark.comet.exec.shuffle.enabled": "true",
- "spark.memory.offHeap.enabled": "true",
- "spark.comet.exec.shuffle.compression.level": "1",
- "spark.executor.memory": "16g",
- "spark.app.name": "comet benchmark derived from tpch",
- "spark.comet.batchSize": "8192",
- "spark.app.startTime": "1736802464855",
- "spark.comet.exec.shuffle.fallbackToColumnar": "true",
- "spark.serializer.objectStreamReset": "100",
- "spark.driver.host": "10.0.0.118",
- "spark.comet.exec.shuffle.enableFastEncoding": "true",
- "spark.submit.deployMode": "client",
- "spark.driver.port": "33103",
- "spark.comet.scan.impl": "native_comet",
- "spark.driver.extraClassPath": "/home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.executor.cores": "8",
- "spark.comet.explain.verbose.enabled": "false",
- "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
- "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
- "spark.comet.exec.enabled": "true",
- "spark.sql.warehouse.dir": "file:/home/andy/git/personal/research/benchmarks/spark-standalone/spark-warehouse",
- "spark.comet.scan.enabled": "true",
- "spark.app.submitTime": "1736802464584",
- "spark.executor.id": "driver",
- "spark.master": "spark://woody:7077",
- "spark.comet.exec.shuffle.mode": "auto",
- "spark.sql.extensions": "org.apache.comet.CometSparkSessionExtensions",
- "spark.driver.memory": "8G",
- "spark.repl.local.jars": "file:///home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.app.initial.jar.urls": "spark://10.0.0.118:33103/jars/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.app.id": "app-20250113140745-0058",
- "spark.rdd.compress": "True",
- "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
- "spark.executor.instances": "1",
- "spark.cores.max": "8",
- "spark.comet.enabled": "true",
- "spark.submit.pyFiles": "",
- "spark.comet.exec.sortMergeJoinWithJoinFilter.enabled": "false",
- "spark.comet.exec.shuffle.compression.codec": "lz4",
- "spark.jars": "file:///home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar",
- "spark.memory.offHeap.size": "16g",
- "spark.comet.columnar.shuffle.batch.size": "8192"
- },
- "1": [
- 12.59755539894104,
- 10.855465650558472,
- 11.160947799682617,
- 11.323237657546997,
- 11.410452365875244
- ],
- "2": [
- 6.155406475067139,
- 5.539891719818115,
- 5.698071002960205,
- 5.684133529663086,
- 5.742799758911133
- ],
- "3": [
- 16.097025156021118,
- 14.982890367507935,
- 14.998259544372559,
- 15.659432649612427,
- 15.878185749053955
- ],
- "4": [
- 10.319517850875854,
- 10.0553297996521,
- 10.136846780776978,
- 9.925675392150879,
- 10.140193462371826
- ],
- "5": [
- 26.09030055999756,
- 25.57556390762329,
- 26.102373600006104,
- 26.540887117385864,
- 26.162983655929565
- ],
- "6": [
- 2.691145658493042,
- 2.5986382961273193,
- 2.659151792526245,
- 2.6488683223724365,
- 2.6785433292388916
- ],
- "7": [
- 15.326677560806274,
- 15.57035493850708,
- 16.023503065109253,
- 16.015883207321167,
- 15.79127025604248
- ],
- "8": [
- 27.72478675842285,
- 27.45163321495056,
- 27.935590267181396,
- 27.86525869369507,
- 28.016165733337402
- ],
- "9": [
- 39.186867237091064,
- 39.73552465438843,
- 40.866581439971924,
- 40.73869442939758,
- 40.89244842529297
- ],
- "10": [
- 14.022773742675781,
- 14.476953029632568,
- 14.305155515670776,
- 14.187727451324463,
- 14.57831335067749
- ],
- "11": [
- 5.223851919174194,
- 4.722897291183472,
- 4.844727277755737,
- 4.803720474243164,
- 4.822873592376709
- ],
- "12": [
- 4.974349021911621,
- 5.013054132461548,
- 5.0682995319366455,
- 5.1071436405181885,
- 5.142468452453613
- ],
- "13": [
- 9.769477128982544,
- 9.743404626846313,
- 9.935744285583496,
- 9.966437339782715,
- 9.854998588562012
- ],
- "14": [
- 5.320314168930054,
- 5.26824426651001,
- 5.269179344177246,
- 5.322073698043823,
- 5.292902708053589
- ],
- "15": [
- 9.532674789428711,
- 9.520610570907593,
- 9.538906335830688,
- 9.553953886032104,
- 9.65409803390503
- ],
- "16": [
- 5.146467924118042,
- 4.716687440872192,
- 4.863113164901733,
- 4.725494384765625,
- 4.653785228729248
- ],
- "17": [
- 30.45087242126465,
- 30.785797119140625,
- 30.950777530670166,
- 31.04833745956421,
- 31.12831139564514
- ],
- "18": [
- 27.549716472625732,
- 27.610363960266113,
- 27.41417407989502,
- 27.633289098739624,
- 27.72838020324707
- ],
- "19": [
- 5.9813477993011475,
- 6.041543483734131,
- 6.087557554244995,
- 6.106397390365601,
- 6.011293888092041
- ],
- "20": [
- 10.53919005393982,
- 10.382107019424438,
- 10.370867729187012,
- 10.376642942428589,
- 10.48800802230835
- ],
- "21": [
- 42.36113142967224,
- 42.296979904174805,
- 42.56899857521057,
- 42.587459564208984,
- 42.86927652359009
- ],
- "22": [
- 3.755877733230591,
- 3.523585319519043,
- 3.5420711040496826,
- 3.605468273162842,
- 3.6084585189819336
- ]
-}
\ No newline at end of file
diff --git a/_downloads/b21ffbbe822a491f187eb8426b042072/comet-tpch.json b/_downloads/b21ffbbe822a491f187eb8426b042072/comet-tpch.json
new file mode 100644
index 000000000..0885a66c7
--- /dev/null
+++ b/_downloads/b21ffbbe822a491f187eb8426b042072/comet-tpch.json
@@ -0,0 +1,134 @@
+{
+ "engine": "datafusion-comet",
+ "benchmark": "tpch",
+ "data_path": "/mnt/bigdata/tpch/sf100/",
+ "query_path": "/tpch",
+ "spark_conf": {
+ "spark.comet.cast.allowIncompatible": "true",
+ "spark.driver.host": "tpcbench-py-7412f095aa6cfc38-driver-svc.default.svc",
+ "spark.kubernetes.memoryOverheadFactor": "0.4",
+ "spark.kubernetes.driver.volumes.hostPath.data.mount.path": "/mnt/bigdata",
+ "spark.kubernetes.executor.container.image": "woody:5000/spark-tpcbench:comet-0.7.0",
+ "spark.executor.extraClassPath": "local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.master": "k8s://https://kube1:6443",
+ "spark.driver.extraClassPath": "local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.driver.bindAddress": "10.42.0.21",
+ "spark.kubernetes.driver.volumes.hostPath.data.options.path": "/mnt/bigdata",
+ "spark.eventLog.enabled": "false",
+ "spark.comet.scan.impl": "native_comet",
+ "spark.kubernetes.container.image.pullPolicy": "Always",
+ "spark.kubernetes.submitInDriver": "true",
+ "spark.app.initial.jar.urls": "file:/opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.sql.warehouse.dir": "file:/opt/spark/work-dir/spark-warehouse",
+ "spark.kubernetes.driver.container.image": "woody:5000/spark-tpcbench:comet-0.7.0",
+ "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
+ "spark.comet.exec.enabled": "true",
+ "spark.executor.id": "driver",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly": "false",
+ "spark.comet.exec.shuffle.mode": "auto",
+ "spark.driver.memory": "8G",
+ "spark.driver.port": "7078",
+ "spark.sql.extensions": "org.apache.comet.CometSparkSessionExtensions",
+ "spark.kubernetes.driver.volumes.hostPath.data.options.readOnly": "false",
+ "spark.kubernetes.executor.volumes.hostPath.data.mount.path": "/mnt/bigdata",
+ "spark.executor.instances": "1",
+ "spark.memory.offHeap.size": "16g",
+ "spark.comet.explain.native.enabled": "false",
+ "spark.app.id": "spark-8f6975ae2c954b5c8179d2190283a150",
+ "spark.kubernetes.authenticate.driver.serviceAccountName": "spark",
+ "spark.kubernetes.driver.pod.name": "tpcbench-py-7412f095aa6cfc38-driver",
+ "spark.executorEnv.SPARK_DRIVER_POD_IP": "10.42.0.21",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName": "benchmark-results-pvc",
+ "spark.comet.exec.replaceSortMergeJoin": "true",
+ "spark.comet.exec.shuffle.enabled": "true",
+ "spark.memory.offHeap.enabled": "true",
+ "spark.comet.exec.shuffle.compression.level": "1",
+ "spark.repl.local.jars": "local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.app.name": "comet benchmark derived from tpch",
+ "spark.executor.memory": "16G",
+ "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
+ "spark.comet.exec.shuffle.fallbackToColumnar": "true",
+ "spark.serializer.objectStreamReset": "100",
+ "spark.app.startTime": "1742321026427",
+ "spark.submit.deployMode": "client",
+ "spark.executor.cores": "8",
+ "spark.kubernetes.executor.podNamePrefix": "comet-benchmark-derived-from-tpch-bcd6cd95aa6d06fc",
+ "spark.app.submitTime": "1742321026189",
+ "spark.jars": "local:/opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.driver.blockManager.port": "7079",
+ "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
+ "spark.plugins": "org.apache.spark.CometPlugin",
+ "spark.kubernetes.resource.type": "python",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path": "/mnt/benchmark-results",
+ "spark.rdd.compress": "True",
+ "spark.kubernetes.executor.volumes.hostPath.data.options.path": "/mnt/bigdata",
+ "spark.submit.pyFiles": "",
+ "spark.comet.exec.shuffle.compression.codec": "lz4"
+ },
+ "1": [
+ 11.800965785980225
+ ],
+ "2": [
+ 4.9119133949279785
+ ],
+ "3": [
+ 9.654481410980225
+ ],
+ "4": [
+ 8.955886125564575
+ ],
+ "5": [
+ 23.40088939666748
+ ],
+ "6": [
+ 2.544438123703003
+ ],
+ "7": [
+ 13.0323166847229
+ ],
+ "8": [
+ 20.596960306167603
+ ],
+ "9": [
+ 30.38415265083313
+ ],
+ "10": [
+ 13.58983588218689
+ ],
+ "11": [
+ 3.9095022678375244
+ ],
+ "12": [
+ 4.531698226928711
+ ],
+ "13": [
+ 7.79347825050354
+ ],
+ "14": [
+ 4.756523370742798
+ ],
+ "15": [
+ 8.379600524902344
+ ],
+ "16": [
+ 5.065413475036621
+ ],
+ "17": [
+ 29.910176277160645
+ ],
+ "18": [
+ 26.39119863510132
+ ],
+ "19": [
+ 5.450929641723633
+ ],
+ "20": [
+ 9.520207166671753
+ ],
+ "21": [
+ 41.77514100074768
+ ],
+ "22": [
+ 4.059680700302124
+ ]
+}
\ No newline at end of file
diff --git a/_downloads/d26631eef01e8ad0873c1990a343fb75/spark-tpcds.json b/_downloads/d26631eef01e8ad0873c1990a343fb75/spark-tpcds.json
deleted file mode 100644
index f3e3b6c32..000000000
--- a/_downloads/d26631eef01e8ad0873c1990a343fb75/spark-tpcds.json
+++ /dev/null
@@ -1,722 +0,0 @@
-{
- "engine": "datafusion-comet",
- "benchmark": "tpcds",
- "data_path": "/mnt/bigdata/tpcds/sf100/",
- "query_path": "../../tpcds/",
- "spark_conf": {
- "spark.eventLog.enabled": "true",
- "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
- "spark.sql.warehouse.dir": "file:/home/andy/git/personal/research/benchmarks/spark-standalone/spark-warehouse",
- "spark.cores.max": "16",
- "spark.executor.id": "driver",
- "spark.master": "spark://woody:7077",
- "spark.app.submitTime": "1736815365737",
- "spark.driver.memory": "8G",
- "spark.executor.memory": "32G",
- "spark.app.startTime": "1736815366015",
- "spark.rdd.compress": "True",
- "spark.executor.instances": "2",
- "spark.app.id": "app-20250113174246-0062",
- "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
- "spark.serializer.objectStreamReset": "100",
- "spark.driver.host": "10.0.0.118",
- "spark.driver.port": "43797",
- "spark.submit.pyFiles": "",
- "spark.submit.deployMode": "client",
- "spark.app.name": "spark benchmark derived from tpcds",
- "spark.executor.cores": "8"
- },
- "1": [
- 5.2619593143463135,
- 1.960148811340332,
- 2.1967520713806152,
- 2.1494410037994385,
- 1.9935636520385742
- ],
- "2": [
- 3.4253926277160645,
- 2.6879563331604004,
- 2.561516046524048,
- 2.5716545581817627,
- 2.563164710998535
- ],
- "3": [
- 1.5403835773468018,
- 1.4260897636413574,
- 1.2098140716552734,
- 1.2134122848510742,
- 1.208524227142334
- ],
- "4": [
- 56.53691649436951,
- 56.77414393424988,
- 56.02755045890808,
- 56.025816440582275,
- 56.88295769691467
- ],
- "5": [
- 10.489055871963501,
- 9.67568039894104,
- 9.388097763061523,
- 9.27160382270813,
- 9.536203384399414
- ],
- "6": [
- 2.1579461097717285,
- 1.9873638153076172,
- 1.9221436977386475,
- 1.9461965560913086,
- 2.0963733196258545
- ],
- "7": [
- 3.032843828201294,
- 2.7287216186523438,
- 2.9177908897399902,
- 2.675243616104126,
- 2.7577996253967285
- ],
- "8": [
- 2.2934205532073975,
- 2.0310394763946533,
- 1.994276762008667,
- 1.9520981311798096,
- 2.4081742763519287
- ],
- "9": [
- 5.7878358364105225,
- 5.4902870655059814,
- 5.461946725845337,
- 5.400648593902588,
- 5.5186686515808105
- ],
- "10": [
- 2.3901453018188477,
- 2.212461233139038,
- 2.298922538757324,
- 2.1111397743225098,
- 2.4135324954986572
- ],
- "11": [
- 17.00469183921814,
- 17.168133020401,
- 15.51388955116272,
- 14.93636679649353,
- 15.758571863174438
- ],
- "12": [
- 1.1017069816589355,
- 0.9583470821380615,
- 0.9638540744781494,
- 0.9336667060852051,
- 1.1460154056549072
- ],
- "13": [
- 4.34652304649353,
- 4.188030481338501,
- 3.886648416519165,
- 3.856201171875,
- 4.094677209854126
- ],
- "14": [
- 44.88281464576721,
- 44.40218901634216,
- 43.64497995376587,
- 43.865254640579224,
- 44.34578776359558
- ],
- "15": [
- 3.323901414871216,
- 3.366013765335083,
- 3.1583144664764404,
- 3.360898971557617,
- 3.554365396499634
- ],
- "16": [
- 12.555872917175293,
- 13.052572965621948,
- 12.834516048431396,
- 12.673231363296509,
- 12.716522216796875
- ],
- "17": [
- 4.7320332527160645,
- 4.594316720962524,
- 4.3970723152160645,
- 4.33561897277832,
- 4.495761871337891
- ],
- "18": [
- 3.5807242393493652,
- 3.535151481628418,
- 3.6561989784240723,
- 3.8135147094726562,
- 3.7007369995117188
- ],
- "19": [
- 2.4516489505767822,
- 2.5885403156280518,
- 2.571218729019165,
- 2.6798818111419678,
- 2.587368965148926
- ],
- "20": [
- 1.1769890785217285,
- 1.084374189376831,
- 1.1451663970947266,
- 1.0745131969451904,
- 1.0696895122528076
- ],
- "21": [
- 2.2724456787109375,
- 1.6616766452789307,
- 2.0807278156280518,
- 1.9577069282531738,
- 1.7267327308654785
- ],
- "22": [
- 21.127835512161255,
- 21.504271745681763,
- 20.774054765701294,
- 20.130521297454834,
- 21.08124542236328
- ],
- "23": [
- 104.4822952747345,
- 102.66089034080505,
- 101.6080904006958,
- 101.47286200523376,
- 102.44011735916138
- ],
- "24": [
- 15.568699359893799,
- 15.30924916267395,
- 14.502421855926514,
- 14.607846736907959,
- 15.179453611373901
- ],
- "25": [
- 4.512059450149536,
- 4.125646352767944,
- 4.210487365722656,
- 4.141698598861694,
- 4.136176586151123
- ],
- "26": [
- 2.0296761989593506,
- 1.7046644687652588,
- 1.8830039501190186,
- 1.9740185737609863,
- 1.7992558479309082
- ],
- "27": [
- 2.637375831604004,
- 2.3286960124969482,
- 2.4201674461364746,
- 2.5083210468292236,
- 2.554165840148926
- ],
- "28": [
- 8.454154253005981,
- 8.10464596748352,
- 8.339635848999023,
- 8.20319676399231,
- 8.14726972579956
- ],
- "29": [
- 5.196781158447266,
- 4.76829719543457,
- 4.92963433265686,
- 4.789939880371094,
- 4.786316633224487
- ],
- "30": [
- 2.0309393405914307,
- 1.8231163024902344,
- 2.189394235610962,
- 1.9110958576202393,
- 1.9947915077209473
- ],
- "31": [
- 5.668262720108032,
- 5.5171403884887695,
- 5.473530292510986,
- 5.337523460388184,
- 5.736505746841431
- ],
- "32": [
- 1.852400779724121,
- 1.6409556865692139,
- 1.7917094230651855,
- 1.6856303215026855,
- 1.6647214889526367
- ],
- "33": [
- 3.3466131687164307,
- 2.9173898696899414,
- 2.859670639038086,
- 2.8988308906555176,
- 2.7332468032836914
- ],
- "34": [
- 1.7460660934448242,
- 1.8845741748809814,
- 1.580057144165039,
- 1.7185635566711426,
- 1.5153119564056396
- ],
- "35": [
- 3.7872586250305176,
- 3.661235809326172,
- 3.492677927017212,
- 3.553354501724243,
- 3.6253058910369873
- ],
- "36": [
- 2.4619619846343994,
- 2.133469343185425,
- 2.03787899017334,
- 2.0881924629211426,
- 2.1240320205688477
- ],
- "37": [
- 4.0605480670928955,
- 3.6801133155822754,
- 3.9225871562957764,
- 3.9400532245635986,
- 3.707016944885254
- ],
- "38": [
- 6.2957329750061035,
- 5.905467987060547,
- 5.979095220565796,
- 7.394719839096069,
- 5.790559530258179
- ],
- "39": [
- 8.440297603607178,
- 8.22698163986206,
- 8.441386938095093,
- 8.33523440361023,
- 8.563800573348999
- ],
- "40": [
- 7.396990537643433,
- 7.279749631881714,
- 7.595478773117065,
- 7.514429807662964,
- 7.445756912231445
- ],
- "41": [
- 0.23249292373657227,
- 0.20892643928527832,
- 0.23296880722045898,
- 0.2194194793701172,
- 0.21021556854248047
- ],
- "42": [
- 1.1865589618682861,
- 1.1890931129455566,
- 1.2970597743988037,
- 1.2622613906860352,
- 1.2915732860565186
- ],
- "43": [
- 1.297257900238037,
- 1.3796677589416504,
- 1.23530912399292,
- 1.2672350406646729,
- 1.2549505233764648
- ],
- "44": [
- 0.941680908203125,
- 0.8873088359832764,
- 0.9511497020721436,
- 0.949620246887207,
- 0.9585254192352295
- ],
- "45": [
- 2.0431082248687744,
- 1.9389359951019287,
- 1.8346209526062012,
- 1.8326334953308105,
- 1.9307496547698975
- ],
- "46": [
- 2.762038230895996,
- 2.790123224258423,
- 2.7187981605529785,
- 2.74556303024292,
- 2.5963001251220703
- ],
- "47": [
- 4.917082786560059,
- 4.889256000518799,
- 5.025712728500366,
- 4.852914333343506,
- 4.931396484375
- ],
- "48": [
- 8.931923627853394,
- 8.777977705001831,
- 8.91053819656372,
- 8.892403841018677,
- 8.816314935684204
- ],
- "49": [
- 7.473886251449585,
- 7.176199197769165,
- 7.320720434188843,
- 7.655316352844238,
- 7.141919136047363
- ],
- "50": [
- 10.274503707885742,
- 10.186567068099976,
- 10.226274490356445,
- 10.341506004333496,
- 10.175342321395874
- ],
- "51": [
- 13.496302127838135,
- 13.125258922576904,
- 13.10242748260498,
- 13.413541078567505,
- 13.212175369262695
- ],
- "52": [
- 1.303222894668579,
- 1.4358396530151367,
- 1.4652376174926758,
- 1.3406119346618652,
- 1.5421879291534424
- ],
- "53": [
- 1.8200466632843018,
- 1.7202496528625488,
- 1.773848533630371,
- 1.7078053951263428,
- 1.649254322052002
- ],
- "54": [
- 3.2271203994750977,
- 3.08542799949646,
- 3.0598156452178955,
- 3.265996217727661,
- 2.9859659671783447
- ],
- "55": [
- 1.4394917488098145,
- 1.4950504302978516,
- 1.499817132949829,
- 1.3825452327728271,
- 1.3539316654205322
- ],
- "56": [
- 2.7544610500335693,
- 2.6781766414642334,
- 2.6310033798217773,
- 2.745976209640503,
- 2.5964338779449463
- ],
- "57": [
- 3.1029934883117676,
- 3.2437119483947754,
- 3.088554859161377,
- 2.882737636566162,
- 3.0023646354675293
- ],
- "58": [
- 3.291809558868408,
- 3.0720701217651367,
- 3.1128735542297363,
- 2.799436569213867,
- 3.03535532951355
- ],
- "59": [
- 3.6170177459716797,
- 3.502054214477539,
- 3.6569032669067383,
- 3.454932689666748,
- 3.7024710178375244
- ],
- "60": [
- 2.3729071617126465,
- 2.374262809753418,
- 2.363983392715454,
- 2.4041998386383057,
- 2.2664554119110107
- ],
- "61": [
- 3.4571456909179688,
- 3.543290138244629,
- 3.4700188636779785,
- 3.515277147293091,
- 3.5213046073913574
- ],
- "62": [
- 1.141568899154663,
- 1.1349658966064453,
- 1.154313087463379,
- 1.1754529476165771,
- 1.1065878868103027
- ],
- "63": [
- 1.536681890487671,
- 1.5299901962280273,
- 1.5405244827270508,
- 1.6240346431732178,
- 1.5590183734893799
- ],
- "64": [
- 22.64835238456726,
- 21.65506911277771,
- 22.328132152557373,
- 22.563674449920654,
- 22.57821035385132
- ],
- "65": [
- 10.918822765350342,
- 10.771467447280884,
- 10.830023527145386,
- 10.823500871658325,
- 10.827515602111816
- ],
- "66": [
- 3.189873695373535,
- 3.2264277935028076,
- 3.0621445178985596,
- 3.1594581604003906,
- 3.149355173110962
- ],
- "67": [
- 80.8468451499939,
- 79.63782620429993,
- 79.21770143508911,
- 80.91981196403503,
- 79.78550696372986
- ],
- "68": [
- 2.941169500350952,
- 2.8041577339172363,
- 2.956421375274658,
- 2.7988245487213135,
- 2.732013463973999
- ],
- "69": [
- 2.204758405685425,
- 2.115067720413208,
- 2.287461280822754,
- 2.1623363494873047,
- 2.3425257205963135
- ],
- "70": [
- 2.833672523498535,
- 2.9473845958709717,
- 2.8611886501312256,
- 2.8069379329681396,
- 2.874908685684204
- ],
- "71": [
- 2.9525787830352783,
- 2.8629682064056396,
- 2.8824713230133057,
- 2.9291234016418457,
- 2.9217307567596436
- ],
- "72": [
- 92.81920456886292,
- 90.03170561790466,
- 88.50960922241211,
- 88.13007974624634,
- 89.58278608322144
- ],
- "73": [
- 1.395726203918457,
- 1.3789515495300293,
- 1.3403804302215576,
- 1.4275565147399902,
- 1.6597483158111572
- ],
- "74": [
- 20.643322706222534,
- 18.648181676864624,
- 20.02110981941223,
- 20.94976043701172,
- 18.866977214813232
- ],
- "75": [
- 12.679317951202393,
- 12.062625408172607,
- 12.371514558792114,
- 11.771496057510376,
- 11.608371257781982
- ],
- "76": [
- 3.0363106727600098,
- 3.0028562545776367,
- 3.063755750656128,
- 2.8598508834838867,
- 3.0945818424224854
- ],
- "77": [
- 3.665741443634033,
- 3.4514997005462646,
- 3.597266435623169,
- 3.332911968231201,
- 3.495286464691162
- ],
- "78": [
- 31.306552410125732,
- 31.237448930740356,
- 32.20013928413391,
- 30.75121283531189,
- 30.8237726688385
- ],
- "79": [
- 2.084214925765991,
- 2.194493532180786,
- 2.1290676593780518,
- 2.12060284614563,
- 2.1929309368133545
- ],
- "80": [
- 25.5107364654541,
- 26.140331983566284,
- 25.496628046035767,
- 26.28699231147766,
- 25.533350229263306
- ],
- "81": [
- 2.231703996658325,
- 2.351255416870117,
- 2.2447509765625,
- 2.3641414642333984,
- 2.3690547943115234
- ],
- "82": [
- 7.771524667739868,
- 7.738788366317749,
- 7.6906514167785645,
- 7.706395387649536,
- 7.73229455947876
- ],
- "83": [
- 1.6065571308135986,
- 1.6151466369628906,
- 1.8333814144134521,
- 1.5380127429962158,
- 1.8119380474090576
- ],
- "84": [
- 1.2147200107574463,
- 1.3057491779327393,
- 1.379417896270752,
- 1.3735690116882324,
- 1.3126640319824219
- ],
- "85": [
- 4.671465635299683,
- 5.442366600036621,
- 4.581593990325928,
- 4.7632105350494385,
- 4.413646936416626
- ],
- "86": [
- 1.0525786876678467,
- 0.9334254264831543,
- 0.8857343196868896,
- 0.9687070846557617,
- 0.943767786026001
- ],
- "87": [
- 6.769503355026245,
- 6.3070619106292725,
- 6.363903284072876,
- 6.062292098999023,
- 6.507101058959961
- ],
- "88": [
- 5.166539907455444,
- 5.316728830337524,
- 5.028919696807861,
- 4.994849681854248,
- 5.05009126663208
- ],
- "89": [
- 1.8866407871246338,
- 1.8803751468658447,
- 1.8070712089538574,
- 1.9447128772735596,
- 1.8955137729644775
- ],
- "90": [
- 0.7426741123199463,
- 0.7263903617858887,
- 0.8066353797912598,
- 0.857090950012207,
- 0.7823832035064697
- ],
- "91": [
- 0.8765714168548584,
- 1.0066447257995605,
- 0.9977982044219971,
- 0.9710574150085449,
- 1.0417554378509521
- ],
- "92": [
- 1.2541251182556152,
- 1.2637217044830322,
- 1.277681827545166,
- 1.272780418395996,
- 1.307555913925171
- ],
- "93": [
- 22.15599775314331,
- 21.807148694992065,
- 21.700384855270386,
- 21.663914918899536,
- 21.595760583877563
- ],
- "94": [
- 6.799180030822754,
- 6.733048439025879,
- 6.928405523300171,
- 6.735382318496704,
- 6.847370147705078
- ],
- "95": [
- 27.122989416122437,
- 27.059486389160156,
- 26.826467514038086,
- 26.69485092163086,
- 27.168644428253174
- ],
- "96": [
- 0.7550704479217529,
- 0.7472677230834961,
- 0.6998603343963623,
- 0.7055039405822754,
- 0.7615151405334473
- ],
- "97": [
- 9.790446996688843,
- 9.716257095336914,
- 9.671125650405884,
- 9.580909967422485,
- 9.678182125091553
- ],
- "98": [
- 2.175020933151245,
- 2.2659518718719482,
- 2.1904609203338623,
- 2.0544354915618896,
- 2.0489299297332764
- ],
- "99": [
- 1.5075435638427734,
- 1.5963904857635498,
- 1.3782060146331787,
- 1.4856946468353271,
- 1.3421602249145508
- ]
-}
\ No newline at end of file
diff --git a/_downloads/eb4015c41cfdc9c2fc70045fbf7f0c84/spark-tpch.json b/_downloads/eb4015c41cfdc9c2fc70045fbf7f0c84/spark-tpch.json
deleted file mode 100644
index a82ea1953..000000000
--- a/_downloads/eb4015c41cfdc9c2fc70045fbf7f0c84/spark-tpch.json
+++ /dev/null
@@ -1,185 +0,0 @@
-{
- "engine": "datafusion-comet",
- "benchmark": "tpch",
- "data_path": "/mnt/bigdata/tpch/sf100/",
- "query_path": "/home/andy/git/apache/datafusion-benchmarks/tpch/queries",
- "spark_conf": {
- "spark.eventLog.enabled": "true",
- "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
- "spark.sql.warehouse.dir": "file:/home/andy/git/personal/research/benchmarks/spark-standalone/spark-warehouse",
- "spark.executor.id": "driver",
- "spark.master": "spark://woody:7077",
- "spark.driver.memory": "8G",
- "spark.memory.offHeap.enabled": "true",
- "spark.driver.port": "34999",
- "spark.executor.memory": "16g",
- "spark.app.id": "app-20250113143837-0059",
- "spark.app.startTime": "1736804317132",
- "spark.rdd.compress": "True",
- "spark.app.name": "spark benchmark derived from tpch",
- "spark.app.submitTime": "1736804316845",
- "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
- "spark.serializer.objectStreamReset": "100",
- "spark.driver.host": "10.0.0.118",
- "spark.executor.instances": "1",
- "spark.cores.max": "8",
- "spark.submit.pyFiles": "",
- "spark.submit.deployMode": "client",
- "spark.executor.cores": "8",
- "spark.memory.offHeap.size": "16g"
- },
- "1": [
- 80.39240145683289,
- 79.9300479888916,
- 80.08533000946045,
- 79.95083689689636,
- 79.85243964195251
- ],
- "2": [
- 12.361650466918945,
- 12.305850267410278,
- 12.250919818878174,
- 11.97881293296814,
- 11.725982427597046
- ],
- "3": [
- 24.618537187576294,
- 24.87288188934326,
- 24.59152317047119,
- 24.218815803527832,
- 24.44655966758728
- ],
- "4": [
- 22.427627086639404,
- 20.657840728759766,
- 21.091227531433105,
- 21.05953359603882,
- 20.46818709373474
- ],
- "5": [
- 50.51570105552673,
- 49.615350008010864,
- 49.8210015296936,
- 49.847323179244995,
- 49.476715087890625
- ],
- "6": [
- 3.347008466720581,
- 3.21842360496521,
- 3.1940605640411377,
- 3.310222625732422,
- 3.24226713180542
- ],
- "7": [
- 21.848386764526367,
- 21.017245531082153,
- 21.077393054962158,
- 21.38729691505432,
- 21.2225079536438
- ],
- "8": [
- 32.984848499298096,
- 33.09590411186218,
- 32.93970465660095,
- 32.48174071311951,
- 32.516525983810425
- ],
- "9": [
- 76.0926308631897,
- 74.51611924171448,
- 74.34783029556274,
- 74.69494938850403,
- 74.43283152580261
- ],
- "10": [
- 19.82606053352356,
- 19.577861070632935,
- 19.701807498931885,
- 19.774757146835327,
- 19.246259689331055
- ],
- "11": [
- 12.777667760848999,
- 12.865848302841187,
- 12.908553838729858,
- 12.77224063873291,
- 12.639224767684937
- ],
- "12": [
- 13.610698223114014,
- 13.52022099494934,
- 13.459492206573486,
- 13.30645203590393,
- 13.632066011428833
- ],
- "13": [
- 22.276411771774292,
- 22.426157474517822,
- 22.358247995376587,
- 22.300530672073364,
- 22.17827558517456
- ],
- "14": [
- 5.936513662338257,
- 5.709101915359497,
- 5.788166046142578,
- 5.724794626235962,
- 5.8799731731414795
- ],
- "15": [
- 14.949971437454224,
- 14.810836553573608,
- 14.723956823348999,
- 14.827229738235474,
- 14.890477657318115
- ],
- "16": [
- 7.259998559951782,
- 6.874396324157715,
- 7.139331817626953,
- 7.010579586029053,
- 6.851199626922607
- ],
- "17": [
- 59.51790142059326,
- 61.10569620132446,
- 61.5246102809906,
- 62.90809988975525,
- 60.082926988601685
- ],
- "18": [
- 71.04480576515198,
- 71.47302889823914,
- 73.99175548553467,
- 70.6622040271759,
- 73.06630039215088
- ],
- "19": [
- 6.902673006057739,
- 7.26209568977356,
- 6.773332357406616,
- 6.810155868530273,
- 6.643052577972412
- ],
- "20": [
- 10.397570133209229,
- 9.828125,
- 9.871150493621826,
- 9.73079776763916,
- 9.764204502105713
- ],
- "21": [
- 66.58646297454834,
- 65.52946162223816,
- 65.2813880443573,
- 67.38516497612,
- 68.25078797340393
- ],
- "22": [
- 9.243720531463623,
- 9.1110098361969,
- 9.128684282302856,
- 9.107020854949951,
- 9.347189903259277
- ]
-}
\ No newline at end of file
diff --git a/_downloads/ee2240cfdfbfad5d101e90ad0cb08b2c/spark-tpch.json b/_downloads/ee2240cfdfbfad5d101e90ad0cb08b2c/spark-tpch.json
new file mode 100644
index 000000000..143a75292
--- /dev/null
+++ b/_downloads/ee2240cfdfbfad5d101e90ad0cb08b2c/spark-tpch.json
@@ -0,0 +1,116 @@
+{
+ "engine": "datafusion-comet",
+ "benchmark": "tpch",
+ "data_path": "/mnt/bigdata/tpch/sf100/",
+ "query_path": "/tpch",
+ "spark_conf": {
+ "spark.kubernetes.executor.container.image": "apache/spark:3.5.3",
+ "spark.kubernetes.memoryOverheadFactor": "0.4",
+ "spark.kubernetes.authenticate.driver.serviceAccountName": "spark",
+ "spark.kubernetes.driver.volumes.hostPath.data.mount.path": "/mnt/bigdata",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName": "benchmark-results-pvc",
+ "spark.memory.offHeap.enabled": "true",
+ "spark.master": "k8s://https://kube1:6443",
+ "spark.kubernetes.executor.podNamePrefix": "spark-benchmark-derived-from-tpch-0234e395aa73a25e",
+ "spark.kubernetes.driver.pod.name": "tpcbench-py-13b7f595aa73982c-driver",
+ "spark.app.submitTime": "1742321459255",
+ "spark.kubernetes.driver.container.image": "woody:5000/spark-tpcbench:spark",
+ "spark.executor.memory": "16G",
+ "spark.serializer.objectStreamReset": "100",
+ "spark.executorEnv.SPARK_DRIVER_POD_IP": "10.42.0.22",
+ "spark.submit.deployMode": "client",
+ "spark.kubernetes.driver.volumes.hostPath.data.options.path": "/mnt/bigdata",
+ "spark.eventLog.enabled": "false",
+ "spark.kubernetes.container.image.pullPolicy": "Always",
+ "spark.executor.cores": "8",
+ "spark.kubernetes.submitInDriver": "true",
+ "spark.driver.host": "tpcbench-py-13b7f595aa73982c-driver-svc.default.svc",
+ "spark.sql.warehouse.dir": "file:/opt/spark/work-dir/spark-warehouse",
+ "spark.app.id": "spark-145218e270c74e29806e49326e3baf05",
+ "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
+ "spark.driver.blockManager.port": "7079",
+ "spark.executor.id": "driver",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly": "false",
+ "spark.driver.memory": "8G",
+ "spark.driver.port": "7078",
+ "spark.app.startTime": "1742321459492",
+ "spark.kubernetes.driver.volumes.hostPath.data.options.readOnly": "false",
+ "spark.kubernetes.resource.type": "python",
+ "spark.kubernetes.executor.volumes.hostPath.data.mount.path": "/mnt/bigdata",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path": "/mnt/benchmark-results",
+ "spark.rdd.compress": "True",
+ "spark.app.name": "spark benchmark derived from tpch",
+ "spark.kubernetes.executor.volumes.hostPath.data.options.path": "/mnt/bigdata",
+ "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
+ "spark.executor.instances": "1",
+ "spark.submit.pyFiles": "",
+ "spark.driver.bindAddress": "10.42.0.22",
+ "spark.memory.offHeap.size": "16g"
+ },
+ "1": [
+ 76.62182855606079
+ ],
+ "2": [
+ 11.83474326133728
+ ],
+ "3": [
+ 23.514692306518555
+ ],
+ "4": [
+ 19.30381989479065
+ ],
+ "5": [
+ 48.1338255405426
+ ],
+ "6": [
+ 3.1663436889648438
+ ],
+ "7": [
+ 20.831352949142456
+ ],
+ "8": [
+ 32.487468957901
+ ],
+ "9": [
+ 72.43645453453064
+ ],
+ "10": [
+ 19.66127896308899
+ ],
+ "11": [
+ 12.756255149841309
+ ],
+ "12": [
+ 13.445186853408813
+ ],
+ "13": [
+ 21.57943820953369
+ ],
+ "14": [
+ 5.569319009780884
+ ],
+ "15": [
+ 14.83508038520813
+ ],
+ "16": [
+ 7.805614948272705
+ ],
+ "17": [
+ 61.21717166900635
+ ],
+ "18": [
+ 70.7374427318573
+ ],
+ "19": [
+ 6.693265914916992
+ ],
+ "20": [
+ 9.911726713180542
+ ],
+ "21": [
+ 64.79066371917725
+ ],
+ "22": [
+ 8.888522863388062
+ ]
+}
\ No newline at end of file
diff --git a/_downloads/f0466a52bebf9649b5d13f67eba6bf43/comet-tpcds.json b/_downloads/f0466a52bebf9649b5d13f67eba6bf43/comet-tpcds.json
new file mode 100644
index 000000000..86df7364e
--- /dev/null
+++ b/_downloads/f0466a52bebf9649b5d13f67eba6bf43/comet-tpcds.json
@@ -0,0 +1,361 @@
+{
+ "engine": "datafusion-comet",
+ "benchmark": "tpcds",
+ "data_path": "/mnt/bigdata/tpcds/sf100/",
+ "query_path": "/tpcds",
+ "spark_conf": {
+ "spark.comet.cast.allowIncompatible": "true",
+ "spark.kubernetes.memoryOverheadFactor": "0.4",
+ "spark.kubernetes.authenticate.driver.serviceAccountName": "spark",
+ "spark.kubernetes.driver.volumes.hostPath.data.mount.path": "/mnt/bigdata",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName": "benchmark-results-pvc",
+ "spark.comet.exec.shuffle.enabled": "true",
+ "spark.kubernetes.executor.container.image": "woody:5000/spark-tpcbench:comet-0.7.0",
+ "spark.memory.offHeap.enabled": "true",
+ "spark.executor.extraClassPath": "local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.memory.offHeap.size": "32g",
+ "spark.master": "k8s://https://kube1:6443",
+ "spark.repl.local.jars": "local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.kubernetes.executor.podNamePrefix": "comet-benchmark-derived-from-tpcds-edbdd195aafaad20",
+ "spark.driver.extraClassPath": "local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.comet.exec.shuffle.fallbackToColumnar": "true",
+ "spark.kubernetes.driver.pod.name": "tpcbench-py-ca7e3a95aafaa234-driver",
+ "spark.serializer.objectStreamReset": "100",
+ "spark.executor.cores": "16",
+ "spark.submit.deployMode": "client",
+ "spark.kubernetes.driver.volumes.hostPath.data.options.path": "/mnt/bigdata",
+ "spark.eventLog.enabled": "false",
+ "spark.kubernetes.container.image.pullPolicy": "Always",
+ "spark.kubernetes.submitInDriver": "true",
+ "spark.comet.exec.replaceSortMergeJoin": "false",
+ "spark.app.initial.jar.urls": "file:/opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.sql.warehouse.dir": "file:/opt/spark/work-dir/spark-warehouse",
+ "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
+ "spark.kubernetes.driver.container.image": "woody:5000/spark-tpcbench:comet-0.7.0",
+ "spark.jars": "local:/opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar",
+ "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
+ "spark.driver.blockManager.port": "7079",
+ "spark.driver.host": "tpcbench-py-ca7e3a95aafaa234-driver-svc.default.svc",
+ "spark.executor.id": "driver",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly": "false",
+ "spark.executorEnv.SPARK_DRIVER_POD_IP": "10.42.0.32",
+ "spark.driver.memory": "8G",
+ "spark.plugins": "org.apache.spark.CometPlugin",
+ "spark.driver.port": "7078",
+ "spark.sql.extensions": "org.apache.comet.CometSparkSessionExtensions",
+ "spark.driver.bindAddress": "10.42.0.32",
+ "spark.executor.memory": "32g",
+ "spark.kubernetes.driver.volumes.hostPath.data.options.readOnly": "false",
+ "spark.kubernetes.resource.type": "python",
+ "spark.kubernetes.executor.volumes.hostPath.data.mount.path": "/mnt/bigdata",
+ "spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path": "/mnt/benchmark-results",
+ "spark.rdd.compress": "True",
+ "spark.kubernetes.executor.volumes.hostPath.data.options.path": "/mnt/bigdata",
+ "spark.app.submitTime": "1742330309312",
+ "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
+ "spark.executor.instances": "1",
+ "spark.submit.pyFiles": "",
+ "spark.app.name": "comet benchmark derived from tpcds",
+ "spark.comet.exec.sortMergeJoinWithJoinFilter.enabled": "false",
+ "spark.comet.exec.shuffle.compression.codec": "lz4",
+ "spark.app.startTime": "1742330309547",
+ "spark.app.id": "spark-f19e10f23d8240b48cc4ed98ed04fba3"
+ },
+ "1": [
+ 3.8007454872131348
+ ],
+ "2": [
+ 2.934406042098999
+ ],
+ "3": [
+ 1.8077075481414795
+ ],
+ "4": [
+ 38.95960068702698
+ ],
+ "5": [
+ 6.480054140090942
+ ],
+ "6": [
+ 1.7508840560913086
+ ],
+ "7": [
+ 3.1088171005249023
+ ],
+ "8": [
+ 1.7765257358551025
+ ],
+ "9": [
+ 6.612036228179932
+ ],
+ "10": [
+ 2.2242751121520996
+ ],
+ "11": [
+ 14.480247735977173
+ ],
+ "12": [
+ 1.7369463443756104
+ ],
+ "13": [
+ 4.074203729629517
+ ],
+ "14": [
+ 29.123273134231567
+ ],
+ "15": [
+ 4.248904705047607
+ ],
+ "16": [
+ 10.133432388305664
+ ],
+ "17": [
+ 3.7625250816345215
+ ],
+ "18": [
+ 3.8477349281311035
+ ],
+ "19": [
+ 4.500071048736572
+ ],
+ "20": [
+ 2.6818368434906006
+ ],
+ "21": [
+ 1.4147498607635498
+ ],
+ "22": [
+ 7.523956060409546
+ ],
+ "23": [
+ 46.224446296691895
+ ],
+ "24": [
+ 17.637375354766846
+ ],
+ "25": [
+ 4.1354968547821045
+ ],
+ "26": [
+ 1.6679058074951172
+ ],
+ "27": [
+ 3.050835609436035
+ ],
+ "28": [
+ 6.84471321105957
+ ],
+ "29": [
+ 5.636432647705078
+ ],
+ "30": [
+ 1.3323733806610107
+ ],
+ "31": [
+ 8.65419864654541
+ ],
+ "32": [
+ 2.1772098541259766
+ ],
+ "33": [
+ 4.056207180023193
+ ],
+ "34": [
+ 1.6547296047210693
+ ],
+ "35": [
+ 3.6222033500671387
+ ],
+ "36": [
+ 3.8398098945617676
+ ],
+ "37": [
+ 2.759655714035034
+ ],
+ "38": [
+ 3.1616175174713135
+ ],
+ "39": [
+ 23.769543647766113
+ ],
+ "40": [
+ 5.029548406600952
+ ],
+ "41": [
+ 0.1620326042175293
+ ],
+ "42": [
+ 1.332118272781372
+ ],
+ "43": [
+ 1.7391486167907715
+ ],
+ "44": [
+ 2.106876850128174
+ ],
+ "45": [
+ 2.75510835647583
+ ],
+ "46": [
+ 3.896399974822998
+ ],
+ "47": [
+ 8.290078401565552
+ ],
+ "48": [
+ 2.7061638832092285
+ ],
+ "49": [
+ 7.190724849700928
+ ],
+ "50": [
+ 6.780158996582031
+ ],
+ "51": [
+ 8.215158224105835
+ ],
+ "52": [
+ 1.5169131755828857
+ ],
+ "53": [
+ 1.7289965152740479
+ ],
+ "54": [
+ 4.88456130027771
+ ],
+ "55": [
+ 1.6041936874389648
+ ],
+ "56": [
+ 3.499546527862549
+ ],
+ "57": [
+ 4.1097211837768555
+ ],
+ "58": [
+ 7.262974500656128
+ ],
+ "59": [
+ 4.279415607452393
+ ],
+ "60": [
+ 3.539266586303711
+ ],
+ "61": [
+ 7.6956703662872314
+ ],
+ "62": [
+ 0.9987466335296631
+ ],
+ "63": [
+ 1.657144546508789
+ ],
+ "64": [
+ 16.91126251220703
+ ],
+ "65": [
+ 8.163438081741333
+ ],
+ "66": [
+ 3.2754974365234375
+ ],
+ "67": [
+ 26.075207233428955
+ ],
+ "68": [
+ 4.537346124649048
+ ],
+ "69": [
+ 2.11562442779541
+ ],
+ "70": [
+ 3.6501145362854004
+ ],
+ "71": [
+ 3.4785256385803223
+ ],
+ "72": [
+ 86.01354646682739
+ ],
+ "73": [
+ 1.4168097972869873
+ ],
+ "74": [
+ 10.193686485290527
+ ],
+ "75": [
+ 12.96819543838501
+ ],
+ "76": [
+ 3.2735350131988525
+ ],
+ "77": [
+ 3.9798026084899902
+ ],
+ "78": [
+ 20.82780885696411
+ ],
+ "79": [
+ 3.004102945327759
+ ],
+ "80": [
+ 15.547224760055542
+ ],
+ "81": [
+ 1.5719726085662842
+ ],
+ "82": [
+ 1.9509336948394775
+ ],
+ "83": [
+ 0.9915726184844971
+ ],
+ "84": [
+ 0.8701736927032471
+ ],
+ "85": [
+ 2.0493757724761963
+ ],
+ "86": [
+ 0.8952863216400146
+ ],
+ "87": [
+ 3.469674587249756
+ ],
+ "88": [
+ 5.753418922424316
+ ],
+ "89": [
+ 2.209171772003174
+ ],
+ "90": [
+ 0.48661088943481445
+ ],
+ "91": [
+ 0.5826904773712158
+ ],
+ "92": [
+ 0.966893196105957
+ ],
+ "93": [
+ 8.976980686187744
+ ],
+ "94": [
+ 6.286005020141602
+ ],
+ "95": [
+ 23.718507051467896
+ ],
+ "96": [
+ 0.8083770275115967
+ ],
+ "97": [
+ 6.110299348831177
+ ],
+ "98": [
+ 5.283054828643799
+ ],
+ "99": [
+ 1.5403597354888916
+ ]
+}
\ No newline at end of file
diff --git a/_images/tpcds_allqueries.png b/_images/tpcds_allqueries.png
index 3fd644095..f083797ef 100644
Binary files a/_images/tpcds_allqueries.png and b/_images/tpcds_allqueries.png differ
diff --git a/_images/tpcds_queries_compare.png b/_images/tpcds_queries_compare.png
index f83a0b4b7..a09e10701 100644
Binary files a/_images/tpcds_queries_compare.png and b/_images/tpcds_queries_compare.png differ
diff --git a/_images/tpcds_queries_speedup_abs.png b/_images/tpcds_queries_speedup_abs.png
index 8ecd448c4..d805f1ead 100644
Binary files a/_images/tpcds_queries_speedup_abs.png and b/_images/tpcds_queries_speedup_abs.png differ
diff --git a/_images/tpcds_queries_speedup_rel.png b/_images/tpcds_queries_speedup_rel.png
index 441cd9464..f27bc6a24 100644
Binary files a/_images/tpcds_queries_speedup_rel.png and b/_images/tpcds_queries_speedup_rel.png differ
diff --git a/_images/tpch_allqueries.png b/_images/tpch_allqueries.png
index 0855f7bf2..f6762e66e 100644
Binary files a/_images/tpch_allqueries.png and b/_images/tpch_allqueries.png differ
diff --git a/_images/tpch_queries_compare.png b/_images/tpch_queries_compare.png
index c17c1af15..3deabc525 100644
Binary files a/_images/tpch_queries_compare.png and b/_images/tpch_queries_compare.png differ
diff --git a/_images/tpch_queries_speedup_abs.png b/_images/tpch_queries_speedup_abs.png
index aa8391cc5..1e5e371d1 100644
Binary files a/_images/tpch_queries_speedup_abs.png and b/_images/tpch_queries_speedup_abs.png differ
diff --git a/_images/tpch_queries_speedup_rel.png b/_images/tpch_queries_speedup_rel.png
index 1fce882eb..c675f03dd 100644
Binary files a/_images/tpch_queries_speedup_rel.png and b/_images/tpch_queries_speedup_rel.png differ
diff --git a/_sources/contributor-guide/benchmark-results/tpc-ds.md.txt b/_sources/contributor-guide/benchmark-results/tpc-ds.md.txt
index 012913189..6b1d4a6e7 100644
--- a/_sources/contributor-guide/benchmark-results/tpc-ds.md.txt
+++ b/_sources/contributor-guide/benchmark-results/tpc-ds.md.txt
@@ -25,24 +25,24 @@ and we encourage you to run these benchmarks in your own environments.
The tracking issue for improving TPC-DS performance is [#858](https://github.com/apache/datafusion-comet/issues/858).
-
+
Here is a breakdown showing relative performance of Spark and Comet for each query.
-
+
The following chart shows how much Comet currently accelerates each query from the benchmark in relative terms.
-
+
The following chart shows how much Comet currently accelerates each query from the benchmark in absolute terms.
-
+
The raw results of these benchmarks in JSON format is available here:
-- [Spark](0.5.0/spark-tpcds.json)
-- [Comet](0.5.0/comet-tpcds.json)
+- [Spark](0.7.0/spark-tpcds.json)
+- [Comet](0.7.0/comet-tpcds.json)
# Scripts
@@ -55,52 +55,77 @@ Here are the scripts that were used to generate these results.
$SPARK_HOME/bin/spark-submit \
--master $SPARK_MASTER \
--conf spark.driver.memory=8G \
- --conf spark.executor.memory=32G \
- --conf spark.executor.instances=2 \
- --conf spark.executor.cores=8 \
- --conf spark.cores.max=16 \
- --conf spark.eventLog.enabled=true \
- tpcbench.py \
- --benchmark tpcds \
+ --conf spark.executor.instances=1 \
+ --conf spark.executor.cores=16 \
+ --conf spark.executor.memory=32g \
+ --conf spark.memory.offHeap.enabled=true \
+ --conf spark.memory.offHeap.size=32g \
+ --deploy-mode cluster \
+ --conf spark.kubernetes.container.image.pullPolicy=Always \
+ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
+ --conf spark.kubernetes.driver.container.image=$SPARK_BENCHMARK_IMAGE \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName=benchmark-results-pvc \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path=/mnt/benchmark-results \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly=false \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.mount.path=/mnt/bigdata \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.options.path=/mnt/bigdata \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.options.readOnly=false \
+ --conf spark.kubernetes.executor.container.image=$SPARK_BENCHMARK_IMAGE \
+ --conf spark.kubernetes.executor.volumes.hostPath.data.mount.path=/mnt/bigdata \
+ --conf spark.kubernetes.executor.volumes.hostPath.data.options.path=/mnt/bigdata \
+ --conf spark.eventLog.enabled=false \
+ local:///tpcbench.py \
--name spark \
+ --benchmark tpcds \
--data /mnt/bigdata/tpcds/sf100/ \
- --queries ../../tpcds/ \
- --output . \
- --iterations 5
+ --queries /tpcds \
+ --output /mnt/benchmark-results \
+ --iterations 1
```
## Apache Spark + Comet
```shell
#!/bin/bash
+export COMET_JAR=local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar
$SPARK_HOME/bin/spark-submit \
--master $SPARK_MASTER \
--conf spark.driver.memory=8G \
- --conf spark.executor.instances=2 \
- --conf spark.executor.memory=16G \
- --conf spark.executor.cores=8 \
- --total-executor-cores=16 \
- --conf spark.eventLog.enabled=true \
- --conf spark.driver.maxResultSize=2G \
+ --conf spark.executor.instances=1 \
+ --conf spark.executor.cores=16 \
+ --conf spark.executor.memory=32g \
--conf spark.memory.offHeap.enabled=true \
- --conf spark.memory.offHeap.size=24g \
+ --conf spark.memory.offHeap.size=32g \
+ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
+ --conf spark.comet.exec.shuffle.enabled=true \
+ --conf spark.comet.exec.shuffle.compression.codec=lz4 \
+ --conf spark.comet.exec.replaceSortMergeJoin=false \
+ --conf spark.comet.exec.sortMergeJoinWithJoinFilter.enabled=false \
+ --conf spark.comet.cast.allowIncompatible=true \
+ --conf spark.comet.exec.shuffle.fallbackToColumnar=true \
+ --deploy-mode cluster \
--jars $COMET_JAR \
--conf spark.driver.extraClassPath=$COMET_JAR \
--conf spark.executor.extraClassPath=$COMET_JAR \
--conf spark.plugins=org.apache.spark.CometPlugin \
- --conf spark.comet.enabled=true \
- --conf spark.comet.cast.allowIncompatible=true \
- --conf spark.comet.exec.replaceSortMergeJoin=false \
- --conf spark.comet.exec.shuffle.enabled=true \
- --conf spark.comet.exec.shuffle.mode=auto \
- --conf spark.comet.exec.shuffle.fallbackToColumnar=true \
- --conf spark.comet.exec.shuffle.compression.codec=lz4 \
- --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
- tpcbench.py \
+ --conf spark.kubernetes.container.image.pullPolicy=Always \
+ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
+ --conf spark.kubernetes.driver.container.image=$COMET_BENCHMARK_IMAGE \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName=benchmark-results-pvc \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path=/mnt/benchmark-results \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly=false \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.mount.path=/mnt/bigdata \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.options.path=/mnt/bigdata \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.options.readOnly=false \
+ --conf spark.kubernetes.executor.container.image=$COMET_BENCHMARK_IMAGE \
+ --conf spark.kubernetes.executor.volumes.hostPath.data.mount.path=/mnt/bigdata \
+ --conf spark.kubernetes.executor.volumes.hostPath.data.options.path=/mnt/bigdata \
+ --conf spark.eventLog.enabled=false \
+ local:///tpcbench.py \
--name comet \
--benchmark tpcds \
--data /mnt/bigdata/tpcds/sf100/ \
- --queries ../../tpcds/ \
- --output . \
- --iterations 5
+ --queries /tpcds \
+ --output /mnt/benchmark-results \
+ --iterations 1
```
\ No newline at end of file
diff --git a/_sources/contributor-guide/benchmark-results/tpc-h.md.txt b/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
index d383cae85..1e08319b6 100644
--- a/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
+++ b/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
@@ -25,24 +25,24 @@ and we encourage you to run these benchmarks in your own environments.
The tracking issue for improving TPC-H performance is [#391](https://github.com/apache/datafusion-comet/issues/391).
-
+
Here is a breakdown showing relative performance of Spark and Comet for each query.
-
+
The following chart shows how much Comet currently accelerates each query from the benchmark in relative terms.
-
+
The following chart shows how much Comet currently accelerates each query from the benchmark in absolute terms.
-
+
The raw results of these benchmarks in JSON format is available here:
-- [Spark](0.5.0/spark-tpch.json)
-- [Comet](0.5.0/comet-tpch.json)
+- [Spark](0.7.0/spark-tpch.json)
+- [Comet](0.7.0/comet-tpch.json)
# Scripts
@@ -51,58 +51,85 @@ Here are the scripts that were used to generate these results.
## Apache Spark
```shell
-#!/bin/bash
$SPARK_HOME/bin/spark-submit \
--master $SPARK_MASTER \
--conf spark.driver.memory=8G \
--conf spark.executor.instances=1 \
--conf spark.executor.cores=8 \
- --conf spark.cores.max=8 \
- --conf spark.executor.memory=16g \
+ --conf spark.executor.memory=16G \
--conf spark.memory.offHeap.enabled=true \
--conf spark.memory.offHeap.size=16g \
- --conf spark.eventLog.enabled=true \
- tpcbench.py \
+ --deploy-mode cluster \
+ --conf spark.kubernetes.container.image.pullPolicy=Always \
+ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
+ --conf spark.kubernetes.driver.container.image=$SPARK_BENCHMARK_IMAGE \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName=benchmark-results-pvc \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path=/mnt/benchmark-results \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly=false \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.mount.path=/mnt/bigdata \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.options.path=/mnt/bigdata \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.options.readOnly=false \
+ --conf spark.kubernetes.executor.container.image=$SPARK_BENCHMARK_IMAGE \
+ --conf spark.kubernetes.executor.volumes.hostPath.data.mount.path=/mnt/bigdata \
+ --conf spark.kubernetes.executor.volumes.hostPath.data.options.path=/mnt/bigdata \
+ --conf spark.eventLog.enabled=false \
+ local:///tpcbench.py \
--name spark \
--benchmark tpch \
--data /mnt/bigdata/tpch/sf100/ \
- --queries ../../tpch/queries \
- --output . \
- --iterations 5
-
+ --queries /tpch \
+ --output /mnt/benchmark-results \
+ --iterations 1
```
## Apache Spark + Comet
```shell
-#!/bin/bash
+export COMET_JAR=local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar
$SPARK_HOME/bin/spark-submit \
--master $SPARK_MASTER \
--conf spark.driver.memory=8G \
--conf spark.executor.instances=1 \
--conf spark.executor.cores=8 \
- --conf spark.cores.max=8 \
- --conf spark.executor.memory=16g \
+ --conf spark.executor.memory=16G \
--conf spark.memory.offHeap.enabled=true \
--conf spark.memory.offHeap.size=16g \
+ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
+ --conf spark.comet.exec.enabled=true \
+ --conf spark.comet.scan.impl=native_comet \
+ --conf spark.comet.cast.allowIncompatible=true \
--conf spark.comet.exec.replaceSortMergeJoin=true \
- --conf spark.eventLog.enabled=true \
- --jars $COMET_JAR \
- --driver-class-path $COMET_JAR \
- --conf spark.driver.extraClassPath=$COMET_JAR \
- --conf spark.executor.extraClassPath=$COMET_JAR \
- --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
- --conf spark.comet.enabled=true \
--conf spark.comet.exec.shuffle.enabled=true \
--conf spark.comet.exec.shuffle.mode=auto \
--conf spark.comet.exec.shuffle.fallbackToColumnar=true \
--conf spark.comet.exec.shuffle.compression.codec=lz4 \
- --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
- tpcbench.py \
+ --conf spark.comet.exec.shuffle.compression.level=1 \
+ --conf spark.comet.explain.native.enabled=false \
+ --deploy-mode cluster \
+ --jars $COMET_JAR \
+ --conf spark.driver.extraClassPath=$COMET_JAR \
+ --conf spark.executor.extraClassPath=$COMET_JAR \
+ --conf spark.driver.extraJavaOptions="-Dlog4j.configuration=file:/opt/spark/conf/log4j.properties" \
+ --conf spark.executor.extraJavaOptions="-Dlog4j.configuration=file:/opt/spark/conf/log4j.properties" \
+ --conf spark.plugins=org.apache.spark.CometPlugin \
+ --conf spark.kubernetes.container.image.pullPolicy=Always \
+ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
+ --conf spark.kubernetes.driver.container.image=$COMET_BENCHMARK_IMAGE \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName=benchmark-results-pvc \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path=/mnt/benchmark-results \
+ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly=false \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.mount.path=/mnt/bigdata \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.options.path=/mnt/bigdata \
+ --conf spark.kubernetes.driver.volumes.hostPath.data.options.readOnly=false \
+ --conf spark.kubernetes.executor.container.image=$COMET_BENCHMARK_IMAGE \
+ --conf spark.kubernetes.executor.volumes.hostPath.data.mount.path=/mnt/bigdata \
+ --conf spark.kubernetes.executor.volumes.hostPath.data.options.path=/mnt/bigdata \
+ --conf spark.eventLog.enabled=false \
+ local:///tpcbench.py \
--name comet \
--benchmark tpch \
--data /mnt/bigdata/tpch/sf100/ \
- --queries ../../tpch/queries \
- --output . \
- --iterations 5
+ --queries /tpch \
+ --output /mnt/benchmark-results \
+ --iterations 1
```
\ No newline at end of file
diff --git a/_static/images/benchmark-results/0.2.0/tpcds_allqueries.png b/_static/images/benchmark-results/0.2.0/tpcds_allqueries.png
deleted file mode 100644
index 0262ac853..000000000
Binary files a/_static/images/benchmark-results/0.2.0/tpcds_allqueries.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.2.0/tpcds_queries_compare.png b/_static/images/benchmark-results/0.2.0/tpcds_queries_compare.png
deleted file mode 100644
index 970cabcb3..000000000
Binary files a/_static/images/benchmark-results/0.2.0/tpcds_queries_compare.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.2.0/tpcds_queries_speedup_abs.png b/_static/images/benchmark-results/0.2.0/tpcds_queries_speedup_abs.png
deleted file mode 100644
index 3034ef063..000000000
Binary files a/_static/images/benchmark-results/0.2.0/tpcds_queries_speedup_abs.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.2.0/tpch_allqueries.png b/_static/images/benchmark-results/0.2.0/tpch_allqueries.png
deleted file mode 100644
index da0a86154..000000000
Binary files a/_static/images/benchmark-results/0.2.0/tpch_allqueries.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.2.0/tpch_queries_compare.png b/_static/images/benchmark-results/0.2.0/tpch_queries_compare.png
deleted file mode 100644
index 326819d9c..000000000
Binary files a/_static/images/benchmark-results/0.2.0/tpch_queries_compare.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.2.0/tpch_queries_speedup_rel.png b/_static/images/benchmark-results/0.2.0/tpch_queries_speedup_rel.png
deleted file mode 100644
index cc247f49b..000000000
Binary files a/_static/images/benchmark-results/0.2.0/tpch_queries_speedup_rel.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.3.0/tpcds_allqueries.png b/_static/images/benchmark-results/0.3.0/tpcds_allqueries.png
deleted file mode 100644
index ed82f8326..000000000
Binary files a/_static/images/benchmark-results/0.3.0/tpcds_allqueries.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.3.0/tpcds_queries_compare.png b/_static/images/benchmark-results/0.3.0/tpcds_queries_compare.png
deleted file mode 100644
index c0ddff31e..000000000
Binary files a/_static/images/benchmark-results/0.3.0/tpcds_queries_compare.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.3.0/tpcds_queries_speedup_abs.png b/_static/images/benchmark-results/0.3.0/tpcds_queries_speedup_abs.png
deleted file mode 100644
index e2146caf8..000000000
Binary files a/_static/images/benchmark-results/0.3.0/tpcds_queries_speedup_abs.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.3.0/tpcds_queries_speedup_rel.png b/_static/images/benchmark-results/0.3.0/tpcds_queries_speedup_rel.png
deleted file mode 100644
index b51d00b58..000000000
Binary files a/_static/images/benchmark-results/0.3.0/tpcds_queries_speedup_rel.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.3.0/tpch_allqueries.png b/_static/images/benchmark-results/0.3.0/tpch_allqueries.png
deleted file mode 100644
index 8fbc3d45c..000000000
Binary files a/_static/images/benchmark-results/0.3.0/tpch_allqueries.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.3.0/tpch_queries_compare.png b/_static/images/benchmark-results/0.3.0/tpch_queries_compare.png
deleted file mode 100644
index 42aec60eb..000000000
Binary files a/_static/images/benchmark-results/0.3.0/tpch_queries_compare.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.3.0/tpch_queries_speedup_abs.png b/_static/images/benchmark-results/0.3.0/tpch_queries_speedup_abs.png
deleted file mode 100644
index 7a6058155..000000000
Binary files a/_static/images/benchmark-results/0.3.0/tpch_queries_speedup_abs.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.3.0/tpch_queries_speedup_rel.png b/_static/images/benchmark-results/0.3.0/tpch_queries_speedup_rel.png
deleted file mode 100644
index 68ac56264..000000000
Binary files a/_static/images/benchmark-results/0.3.0/tpch_queries_speedup_rel.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.4.0/tpcds_allqueries.png b/_static/images/benchmark-results/0.4.0/tpcds_allqueries.png
deleted file mode 100644
index b1c9e3c90..000000000
Binary files a/_static/images/benchmark-results/0.4.0/tpcds_allqueries.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.4.0/tpcds_queries_compare.png b/_static/images/benchmark-results/0.4.0/tpcds_queries_compare.png
deleted file mode 100644
index e367fe4cc..000000000
Binary files a/_static/images/benchmark-results/0.4.0/tpcds_queries_compare.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.4.0/tpcds_queries_speedup_abs.png b/_static/images/benchmark-results/0.4.0/tpcds_queries_speedup_abs.png
deleted file mode 100644
index d719bdbb1..000000000
Binary files a/_static/images/benchmark-results/0.4.0/tpcds_queries_speedup_abs.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.4.0/tpcds_queries_speedup_rel.png b/_static/images/benchmark-results/0.4.0/tpcds_queries_speedup_rel.png
deleted file mode 100644
index afbea75ac..000000000
Binary files a/_static/images/benchmark-results/0.4.0/tpcds_queries_speedup_rel.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.4.0/tpch_allqueries.png b/_static/images/benchmark-results/0.4.0/tpch_allqueries.png
deleted file mode 100644
index 24a3698bb..000000000
Binary files a/_static/images/benchmark-results/0.4.0/tpch_allqueries.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.4.0/tpch_queries_compare.png b/_static/images/benchmark-results/0.4.0/tpch_queries_compare.png
deleted file mode 100644
index 8d3223a9f..000000000
Binary files a/_static/images/benchmark-results/0.4.0/tpch_queries_compare.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.4.0/tpch_queries_speedup_abs.png b/_static/images/benchmark-results/0.4.0/tpch_queries_speedup_abs.png
deleted file mode 100644
index f8b3eeaea..000000000
Binary files a/_static/images/benchmark-results/0.4.0/tpch_queries_speedup_abs.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.4.0/tpch_queries_speedup_rel.png b/_static/images/benchmark-results/0.4.0/tpch_queries_speedup_rel.png
deleted file mode 100644
index d95b41503..000000000
Binary files a/_static/images/benchmark-results/0.4.0/tpch_queries_speedup_rel.png and /dev/null differ
diff --git a/_static/images/benchmark-results/0.7.0/tpcds_allqueries.png b/_static/images/benchmark-results/0.7.0/tpcds_allqueries.png
new file mode 100644
index 000000000..f083797ef
Binary files /dev/null and b/_static/images/benchmark-results/0.7.0/tpcds_allqueries.png differ
diff --git a/_static/images/benchmark-results/0.7.0/tpcds_queries_compare.png b/_static/images/benchmark-results/0.7.0/tpcds_queries_compare.png
new file mode 100644
index 000000000..a09e10701
Binary files /dev/null and b/_static/images/benchmark-results/0.7.0/tpcds_queries_compare.png differ
diff --git a/_static/images/benchmark-results/0.7.0/tpcds_queries_speedup_abs.png b/_static/images/benchmark-results/0.7.0/tpcds_queries_speedup_abs.png
new file mode 100644
index 000000000..d805f1ead
Binary files /dev/null and b/_static/images/benchmark-results/0.7.0/tpcds_queries_speedup_abs.png differ
diff --git a/_static/images/benchmark-results/0.7.0/tpcds_queries_speedup_rel.png b/_static/images/benchmark-results/0.7.0/tpcds_queries_speedup_rel.png
new file mode 100644
index 000000000..f27bc6a24
Binary files /dev/null and b/_static/images/benchmark-results/0.7.0/tpcds_queries_speedup_rel.png differ
diff --git a/_static/images/benchmark-results/0.7.0/tpch_allqueries.png b/_static/images/benchmark-results/0.7.0/tpch_allqueries.png
new file mode 100644
index 000000000..f6762e66e
Binary files /dev/null and b/_static/images/benchmark-results/0.7.0/tpch_allqueries.png differ
diff --git a/_static/images/benchmark-results/0.7.0/tpch_queries_compare.png b/_static/images/benchmark-results/0.7.0/tpch_queries_compare.png
new file mode 100644
index 000000000..3deabc525
Binary files /dev/null and b/_static/images/benchmark-results/0.7.0/tpch_queries_compare.png differ
diff --git a/_static/images/benchmark-results/0.7.0/tpch_queries_speedup_abs.png b/_static/images/benchmark-results/0.7.0/tpch_queries_speedup_abs.png
new file mode 100644
index 000000000..1e5e371d1
Binary files /dev/null and b/_static/images/benchmark-results/0.7.0/tpch_queries_speedup_abs.png differ
diff --git a/_static/images/benchmark-results/0.7.0/tpch_queries_speedup_rel.png b/_static/images/benchmark-results/0.7.0/tpch_queries_speedup_rel.png
new file mode 100644
index 000000000..c675f03dd
Binary files /dev/null and b/_static/images/benchmark-results/0.7.0/tpch_queries_speedup_rel.png differ
diff --git a/contributor-guide/benchmark-results/tpc-ds.html b/contributor-guide/benchmark-results/tpc-ds.html
index bdd7c8e34..f0eb53780 100644
--- a/contributor-guide/benchmark-results/tpc-ds.html
+++ b/contributor-guide/benchmark-results/tpc-ds.html
@@ -367,8 +367,8 @@ and we encourage you to run these benchmarks in your own environments.</p>
<p><img alt="" src="../../_images/tpcds_queries_speedup_abs.png" /></p>
<p>The raw results of these benchmarks in JSON format is available here:</p>
<ul class="simple">
-<li><p><a class="reference download internal" download="" href="../../_downloads/d26631eef01e8ad0873c1990a343fb75/spark-tpcds.json"><span class="xref download myst">Spark</span></a></p></li>
-<li><p><a class="reference download internal" download="" href="../../_downloads/a2bf8d2de6f449ff2ab1739c3a17f18e/comet-tpcds.json"><span class="xref download myst">Comet</span></a></p></li>
+<li><p><a class="reference download internal" download="" href="../../_downloads/4cc7a10f09550a16fb1b753e4fd5d0e2/spark-tpcds.json"><span class="xref download myst">Spark</span></a></p></li>
+<li><p><a class="reference download internal" download="" href="../../_downloads/f0466a52bebf9649b5d13f67eba6bf43/comet-tpcds.json"><span class="xref download myst">Comet</span></a></p></li>
</ul>
</section>
<section id="scripts">
@@ -380,54 +380,79 @@ and we encourage you to run these benchmarks in your own environments.</p>
<span class="nv">$SPARK_HOME</span>/bin/spark-submit<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--master<span class="w"> </span><span class="nv">$SPARK_MASTER</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.driver.memory<span class="o">=</span>8G<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.memory<span class="o">=</span>32G<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.instances<span class="o">=</span><span class="m">2</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.cores<span class="o">=</span><span class="m">8</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.cores.max<span class="o">=</span><span class="m">16</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.eventLog.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>tpcbench.py<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--benchmark<span class="w"> </span>tpcds<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.instances<span class="o">=</span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.cores<span class="o">=</span><span class="m">16</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.memory<span class="o">=</span>32g<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.size<span class="o">=</span>32g<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--deploy-mode<span class="w"> </span>cluster<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.container.image.pullPolicy<span class="o">=</span>Always<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.authenticate.driver.serviceAccountName<span class="o">=</span>spark<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.container.image<span class="o">=</span><span class="nv">$SPARK_BENCHMARK_IMAGE</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName<span class="o">=</span>benchmark-results-pvc<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path<span class="o">=</span>/mnt/benchmark-results<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.mount.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.options.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.options.readOnly<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.container.image<span class="o">=</span><span class="nv">$SPARK_BENCHMARK_IMAGE</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.volumes.hostPath.data.mount.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.volumes.hostPath.data.options.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.eventLog.enabled<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>local:///tpcbench.py<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--name<span class="w"> </span>spark<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--benchmark<span class="w"> </span>tpcds<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--data<span class="w"> </span>/mnt/bigdata/tpcds/sf100/<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--queries<span class="w"> </span>../../tpcds/<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--output<span class="w"> </span>.<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--iterations<span class="w"> </span><span class="m">5</span>
+<span class="w"> </span>--queries<span class="w"> </span>/tpcds<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--output<span class="w"> </span>/mnt/benchmark-results<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--iterations<span class="w"> </span><span class="m">1</span>
</pre></div>
</div>
</section>
<section id="apache-spark-comet">
<h2>Apache Spark + Comet<a class="headerlink" href="#apache-spark-comet" title="Link to this heading">¶</a></h2>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="ch">#!/bin/bash</span>
+<span class="nb">export</span><span class="w"> </span><span class="nv">COMET_JAR</span><span class="o">=</span>local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar
<span class="nv">$SPARK_HOME</span>/bin/spark-submit<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--master<span class="w"> </span><span class="nv">$SPARK_MASTER</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.driver.memory<span class="o">=</span>8G<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.instances<span class="o">=</span><span class="m">2</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.memory<span class="o">=</span>16G<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.cores<span class="o">=</span><span class="m">8</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--total-executor-cores<span class="o">=</span><span class="m">16</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.eventLog.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.driver.maxResultSize<span class="o">=</span>2G<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.instances<span class="o">=</span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.cores<span class="o">=</span><span class="m">16</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.memory<span class="o">=</span>32g<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.size<span class="o">=</span>24g<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.size<span class="o">=</span>32g<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.shuffle.manager<span class="o">=</span>org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.compression.codec<span class="o">=</span>lz4<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.replaceSortMergeJoin<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.sortMergeJoinWithJoinFilter.enabled<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.cast.allowIncompatible<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.fallbackToColumnar<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--deploy-mode<span class="w"> </span>cluster<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--jars<span class="w"> </span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.driver.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.executor.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.plugins<span class="o">=</span>org.apache.spark.CometPlugin<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.cast.allowIncompatible<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.replaceSortMergeJoin<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.mode<span class="o">=</span>auto<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.fallbackToColumnar<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.compression.codec<span class="o">=</span>lz4<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.shuffle.manager<span class="o">=</span>org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>tpcbench.py<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.container.image.pullPolicy<span class="o">=</span>Always<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.authenticate.driver.serviceAccountName<span class="o">=</span>spark<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.container.image<span class="o">=</span><span class="nv">$COMET_BENCHMARK_IMAGE</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName<span class="o">=</span>benchmark-results-pvc<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path<span class="o">=</span>/mnt/benchmark-results<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.mount.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.options.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.options.readOnly<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.container.image<span class="o">=</span><span class="nv">$COMET_BENCHMARK_IMAGE</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.volumes.hostPath.data.mount.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.volumes.hostPath.data.options.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.eventLog.enabled<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>local:///tpcbench.py<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--name<span class="w"> </span>comet<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--benchmark<span class="w"> </span>tpcds<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--data<span class="w"> </span>/mnt/bigdata/tpcds/sf100/<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--queries<span class="w"> </span>../../tpcds/<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--output<span class="w"> </span>.<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--iterations<span class="w"> </span><span class="m">5</span>
+<span class="w"> </span>--queries<span class="w"> </span>/tpcds<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--output<span class="w"> </span>/mnt/benchmark-results<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--iterations<span class="w"> </span><span class="m">1</span>
</pre></div>
</div>
</section>
diff --git a/contributor-guide/benchmark-results/tpc-h.html b/contributor-guide/benchmark-results/tpc-h.html
index f6e53d9ed..533892d71 100644
--- a/contributor-guide/benchmark-results/tpc-h.html
+++ b/contributor-guide/benchmark-results/tpc-h.html
@@ -367,8 +367,8 @@ and we encourage you to run these benchmarks in your own environments.</p>
<p><img alt="" src="../../_images/tpch_queries_speedup_abs.png" /></p>
<p>The raw results of these benchmarks in JSON format is available here:</p>
<ul class="simple">
-<li><p><a class="reference download internal" download="" href="../../_downloads/eb4015c41cfdc9c2fc70045fbf7f0c84/spark-tpch.json"><span class="xref download myst">Spark</span></a></p></li>
-<li><p><a class="reference download internal" download="" href="../../_downloads/a57370497e8278995e02e50496e3d7d8/comet-tpch.json"><span class="xref download myst">Comet</span></a></p></li>
+<li><p><a class="reference download internal" download="" href="../../_downloads/ee2240cfdfbfad5d101e90ad0cb08b2c/spark-tpch.json"><span class="xref download myst">Spark</span></a></p></li>
+<li><p><a class="reference download internal" download="" href="../../_downloads/b21ffbbe822a491f187eb8426b042072/comet-tpch.json"><span class="xref download myst">Comet</span></a></p></li>
</ul>
</section>
<section id="scripts">
@@ -376,59 +376,87 @@ and we encourage you to run these benchmarks in your own environments.</p>
<p>Here are the scripts that were used to generate these results.</p>
<section id="apache-spark">
<h2>Apache Spark<a class="headerlink" href="#apache-spark" title="Link to this heading">¶</a></h2>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="ch">#!/bin/bash</span>
-<span class="nv">$SPARK_HOME</span>/bin/spark-submit<span class="w"> </span><span class="se">\</span>
+<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nv">$SPARK_HOME</span>/bin/spark-submit<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--master<span class="w"> </span><span class="nv">$SPARK_MASTER</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.driver.memory<span class="o">=</span>8G<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.executor.instances<span class="o">=</span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.executor.cores<span class="o">=</span><span class="m">8</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.cores.max<span class="o">=</span><span class="m">8</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.memory<span class="o">=</span>16g<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.memory<span class="o">=</span>16G<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.size<span class="o">=</span>16g<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.eventLog.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>tpcbench.py<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--deploy-mode<span class="w"> </span>cluster<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.container.image.pullPolicy<span class="o">=</span>Always<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.authenticate.driver.serviceAccountName<span class="o">=</span>spark<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.container.image<span class="o">=</span><span class="nv">$SPARK_BENCHMARK_IMAGE</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName<span class="o">=</span>benchmark-results-pvc<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path<span class="o">=</span>/mnt/benchmark-results<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.mount.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.options.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.options.readOnly<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.container.image<span class="o">=</span><span class="nv">$SPARK_BENCHMARK_IMAGE</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.volumes.hostPath.data.mount.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.volumes.hostPath.data.options.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.eventLog.enabled<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>local:///tpcbench.py<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--name<span class="w"> </span>spark<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--benchmark<span class="w"> </span>tpch<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--data<span class="w"> </span>/mnt/bigdata/tpch/sf100/<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--queries<span class="w"> </span>../../tpch/queries<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--output<span class="w"> </span>.<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--iterations<span class="w"> </span><span class="m">5</span>
+<span class="w"> </span>--queries<span class="w"> </span>/tpch<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--output<span class="w"> </span>/mnt/benchmark-results<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--iterations<span class="w"> </span><span class="m">1</span>
</pre></div>
</div>
</section>
<section id="apache-spark-comet">
<h2>Apache Spark + Comet<a class="headerlink" href="#apache-spark-comet" title="Link to this heading">¶</a></h2>
-<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="ch">#!/bin/bash</span>
+<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nb">export</span><span class="w"> </span><span class="nv">COMET_JAR</span><span class="o">=</span>local:///opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar
<span class="nv">$SPARK_HOME</span>/bin/spark-submit<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--master<span class="w"> </span><span class="nv">$SPARK_MASTER</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.driver.memory<span class="o">=</span>8G<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.executor.instances<span class="o">=</span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.executor.cores<span class="o">=</span><span class="m">8</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.cores.max<span class="o">=</span><span class="m">8</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.memory<span class="o">=</span>16g<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.memory<span class="o">=</span>16G<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.memory.offHeap.size<span class="o">=</span>16g<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.shuffle.manager<span class="o">=</span>org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.scan.impl<span class="o">=</span>native_comet<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.cast.allowIncompatible<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.replaceSortMergeJoin<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.eventLog.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--jars<span class="w"> </span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--driver-class-path<span class="w"> </span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.driver.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.executor.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.sql.extensions<span class="o">=</span>org.apache.comet.CometSparkSessionExtensions<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.comet.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.enabled<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.mode<span class="o">=</span>auto<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.fallbackToColumnar<span class="o">=</span><span class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.compression.codec<span class="o">=</span>lz4<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w"> </span>spark.shuffle.manager<span class="o">=</span>org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>tpcbench.py<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.exec.shuffle.compression.level<span class="o">=</span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.comet.explain.native.enabled<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--deploy-mode<span class="w"> </span>cluster<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--jars<span class="w"> </span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.driver.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.extraClassPath<span class="o">=</span><span class="nv">$COMET_JAR</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.driver.extraJavaOptions<span class="o">=</span><span class="s2">"-Dlog4j.configuration=file:/opt/spark/conf/log4j.properties"</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.executor.extraJavaOptions<span class="o">=</span><span class="s2">"-Dlog4j.configuration=file:/opt/spark/conf/log4j.properties"</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.plugins<span class="o">=</span>org.apache.spark.CometPlugin<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.container.image.pullPolicy<span class="o">=</span>Always<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.authenticate.driver.serviceAccountName<span class="o">=</span>spark<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.container.image<span class="o">=</span><span class="nv">$COMET_BENCHMARK_IMAGE</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.options.claimName<span class="o">=</span>benchmark-results-pvc<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.path<span class="o">=</span>/mnt/benchmark-results<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.persistentVolumeClaim.benchmark-results-pv.mount.readOnly<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.mount.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.options.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.driver.volumes.hostPath.data.options.readOnly<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.container.image<span class="o">=</span><span class="nv">$COMET_BENCHMARK_IMAGE</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.volumes.hostPath.data.mount.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.kubernetes.executor.volumes.hostPath.data.options.path<span class="o">=</span>/mnt/bigdata<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w"> </span>spark.eventLog.enabled<span class="o">=</span><span class="nb">false</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>local:///tpcbench.py<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--name<span class="w"> </span>comet<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--benchmark<span class="w"> </span>tpch<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--data<span class="w"> </span>/mnt/bigdata/tpch/sf100/<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--queries<span class="w"> </span>../../tpch/queries<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--output<span class="w"> </span>.<span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--iterations<span class="w"> </span><span class="m">5</span>
+<span class="w"> </span>--queries<span class="w"> </span>/tpch<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--output<span class="w"> </span>/mnt/benchmark-results<span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--iterations<span class="w"> </span><span class="m">1</span>
</pre></div>
</div>
</section>
diff --git a/searchindex.js b/searchindex.js
index 5115117f7..4c1e3bc85 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"1. Install Comet": [[9, "install-comet"]], "2. Clone Spark and Apply Diff": [[9, "clone-spark-and-apply-diff"]], "3. Run Spark SQL Tests": [[9, "run-spark-sql-tests"]], "ANSI mode": [[11, "ansi-mode"]], "API Differences Between Spark Versions": [[0, "api-differences-between-spark-versions"]], "ASF Links": [[10, null]], "Adding Spark-side Tests for the New Expression": [[0, "adding-spark-side-tests-for-the-new-expression"]], "Adding a New Expression": [[0, [...]
\ No newline at end of file
+Search.setIndex({"alltitles": {"1. Install Comet": [[9, "install-comet"]], "2. Clone Spark and Apply Diff": [[9, "clone-spark-and-apply-diff"]], "3. Run Spark SQL Tests": [[9, "run-spark-sql-tests"]], "ANSI mode": [[11, "ansi-mode"]], "API Differences Between Spark Versions": [[0, "api-differences-between-spark-versions"]], "ASF Links": [[10, null]], "Adding Spark-side Tests for the New Expression": [[0, "adding-spark-side-tests-for-the-new-expression"]], "Adding a New Expression": [[0, [...]
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org