You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by gi...@apache.org on 2026/10/09 12:58:16 UTC
(gluten-site) branch asf-site updated: Automatic Site Publish by Buildbot
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/gluten-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new b5791cbf Automatic Site Publish by Buildbot
b5791cbf is described below
commit b5791cbf5fa524cef306d4bf8d92ff6d334010bf
Author: buildbot <us...@infra.apache.org>
AuthorDate: Fri Oct 9 12:58:10 2026 +0000
Automatic Site Publish by Buildbot
---
.../10/09/community-over-code-glasgow-2026.html} | 47 +-
output/archives/index.html | 2 +-
.../v1.1.1/developers/cpp_code_style/index.html | 2 +-
.../v1.1.1/developers/docker_centos7/index.html | 2 +-
.../archives/v1.1.1/developers/docker_centos8.html | 2 +-
.../v1.1.1/developers/docker_ubuntu/index.html | 2 +-
output/archives/v1.1.1/developers/howto/index.html | 2 +-
.../v1.1.1/developers/howtorelease/index.html | 2 +-
output/archives/v1.1.1/developers/index.html | 2 +-
.../v1.1.1/developers/microbenchmark/index.html | 2 +-
output/archives/v1.1.1/developers/newto/index.html | 2 +-
.../v1.1.1/developers/substrait/index.html | 2 +-
.../velox_function_development/index.html | 2 +-
.../docs/clickhouse/getting-started/index.html | 2 +-
output/archives/v1.1.1/docs/clickhouse/index.html | 2 +-
output/archives/v1.1.1/docs/index.html | 2 +-
.../v1.1.1/docs/velox/build_parameters/index.html | 2 +-
output/archives/v1.1.1/docs/velox/gcs/index.html | 2 +-
.../v1.1.1/docs/velox/getting-started/index.html | 2 +-
output/archives/v1.1.1/docs/velox/index.html | 2 +-
output/archives/v1.1.1/docs/velox/s3/index.html | 2 +-
output/archives/v1.1.1/index.html | 2 +-
.../archives/v1.2.1/docs/configuration/index.html | 2 +-
.../docs/developers/cpp-code-style/index.html | 2 +-
.../docs/developers/docker-centos7/index.html | 2 +-
.../docs/developers/docker-centos8/index.html | 2 +-
.../docs/developers/docker-ubuntu/index.html | 2 +-
.../docs/developers/how-to-release/index.html | 2 +-
.../developers/how-to-security-scan/index.html | 2 +-
.../v1.2.1/docs/developers/how-to-use/index.html | 2 +-
output/archives/v1.2.1/docs/developers/index.html | 2 +-
.../docs/developers/microbenchmarks/index.html | 2 +-
.../v1.2.1/docs/developers/new-to/index.html | 2 +-
.../v1.2.1/docs/developers/substrait/index.html | 2 +-
.../docs/developers/velox-function-dev/index.html | 2 +-
.../v1.2.1/docs/getting-started/abfs/index.html | 2 +-
.../docs/getting-started/build-guide/index.html | 2 +-
.../getting-started/clickhouse-backend/index.html | 2 +-
.../v1.2.1/docs/getting-started/gcs/index.html | 2 +-
.../v1.2.1/docs/getting-started/index.html | 2 +-
.../docs/getting-started/localcache/index.html | 2 +-
.../v1.2.1/docs/getting-started/s3/index.html | 2 +-
.../docs/getting-started/velox-backend/index.html | 2 +-
output/archives/v1.2.1/docs/index.html | 2 +-
.../archives/v1.2.1/docs/velox-backend/index.html | 2 +-
.../docs/velox-backend/limitations/index.html | 2 +-
.../v1.2.1/docs/velox-backend/support/index.html | 2 +-
.../docs/velox-backend/troubleshooting/index.html | 2 +-
.../v1.2.1/docs/velox-backend/udf/index.html | 2 +-
output/archives/v1.2.1/index.html | 2 +-
output/archives/v1.3.0/configuration.html | 2 +-
output/archives/v1.3.0/developers.html | 2 +-
.../v1.3.0/developers/cpp-code-style/index.html | 2 +-
.../v1.3.0/developers/docker-centos7/index.html | 2 +-
.../v1.3.0/developers/docker-centos8/index.html | 2 +-
.../v1.3.0/developers/docker-ubuntu/index.html | 2 +-
.../developers/how-to-security-scan/index.html | 2 +-
.../v1.3.0/developers/how-to-use/index.html | 2 +-
.../v1.3.0/developers/microbenchmarks/index.html | 2 +-
.../archives/v1.3.0/developers/new-to/index.html | 2 +-
.../v1.3.0/developers/substrait/index.html | 2 +-
.../developers/velox-function-dev/index.html | 2 +-
output/archives/v1.3.0/getting-started.html | 2 +-
.../v1.3.0/getting-started/abfs/index.html | 2 +-
.../v1.3.0/getting-started/build-guide/index.html | 2 +-
.../getting-started/clickhouse-backend/index.html | 2 +-
.../archives/v1.3.0/getting-started/gcs/index.html | 2 +-
.../v1.3.0/getting-started/localcache/index.html | 2 +-
.../archives/v1.3.0/getting-started/s3/index.html | 2 +-
.../getting-started/velox-backend/index.html | 2 +-
output/archives/v1.3.0/index.html | 2 +-
output/archives/v1.3.0/velox-backend.html | 2 +-
.../v1.3.0/velox-backend/limitations/index.html | 2 +-
.../v1.3.0/velox-backend/support/index.html | 2 +-
.../velox-backend/troubleshooting/index.html | 2 +-
.../archives/v1.3.0/velox-backend/udf/index.html | 2 +-
output/archives/v1.4.0/developers.html | 2 +-
output/archives/v1.4.0/getting-started.html | 2 +-
output/archives/v1.4.0/index.html | 2 +-
output/archives/v1.4.0/velox-backend.html | 2 +-
output/archives/v1.5.0/developers.html | 2 +-
output/archives/v1.5.0/getting-started.html | 2 +-
output/archives/v1.5.0/index.html | 2 +-
output/archives/v1.5.0/velox-backend.html | 2 +-
output/assets/js/search-data.json | 2040 ++++++++++----------
output/blog/index.html | 8 +
output/docs/developers/how-to-release/index.html | 2 +-
output/docs/v1.4.0/Configuration.html | 2 +-
output/docs/v1.4.0/developers/CppCodingStyle.html | 2 +-
output/docs/v1.4.0/developers/HowTo.html | 2 +-
output/docs/v1.4.0/developers/MicroBenchmarks.html | 2 +-
output/docs/v1.4.0/developers/NewToGluten.html | 2 +-
.../developers/ProfileMemoryOfGlutenWithVelox.html | 2 +-
output/docs/v1.4.0/developers/QueryTrace.html | 2 +-
.../v1.4.0/developers/SubstraitModifications.html | 2 +-
.../v1.4.0/developers/UsingGperftoolsInCH.html | 2 +-
.../v1.4.0/developers/UsingJemallocWithCH.html | 2 +-
output/docs/v1.4.0/developers/VeloxUDF.html | 2 +-
.../developers/clickhouse-backend-debug.html | 2 +-
.../v1.4.0/developers/improvement-proposals.html | 2 +-
.../docs/v1.4.0/developers/velox-backend-CI.html | 2 +-
.../developers/velox-backend-build-in-docker.html | 2 +-
.../velox-function-development-guide.html | 2 +-
output/docs/v1.4.0/get-started/ClickHouse.html | 2 +-
output/docs/v1.4.0/get-started/GlutenUI.html | 2 +-
output/docs/v1.4.0/get-started/Velox.html | 2 +-
output/docs/v1.4.0/get-started/VeloxABFS.html | 2 +-
output/docs/v1.4.0/get-started/VeloxGCS.html | 2 +-
output/docs/v1.4.0/get-started/VeloxHBM.html | 2 +-
output/docs/v1.4.0/get-started/VeloxIAA.html | 2 +-
output/docs/v1.4.0/get-started/VeloxIceberg.html | 2 +-
.../docs/v1.4.0/get-started/VeloxLocalCache.html | 2 +-
output/docs/v1.4.0/get-started/VeloxQAT.html | 2 +-
output/docs/v1.4.0/get-started/VeloxS3.html | 2 +-
.../v1.4.0/get-started/VeloxStageResourceAdj.html | 2 +-
output/docs/v1.4.0/get-started/build-guide.html | 2 +-
.../velox-backend/velox-backend-limitations.html | 2 +-
.../velox-backend-scalar-function-support.html | 2 +-
.../velox-backend-support-progress.html | 2 +-
.../velox-backend-troubleshooting.html | 2 +-
output/docs/v1.5.0/Configuration.html | 2 +-
output/docs/v1.5.0/developers/CppCodingStyle.html | 2 +-
output/docs/v1.5.0/developers/HowTo.html | 2 +-
output/docs/v1.5.0/developers/MicroBenchmarks.html | 2 +-
output/docs/v1.5.0/developers/NewToGluten.html | 2 +-
output/docs/v1.5.0/developers/PartialProject.html | 2 +-
.../developers/ProfileMemoryOfGlutenWithVelox.html | 2 +-
output/docs/v1.5.0/developers/QueryTrace.html | 2 +-
.../v1.5.0/developers/SubstraitModifications.html | 2 +-
.../v1.5.0/developers/UsingGperftoolsInCH.html | 2 +-
.../v1.5.0/developers/UsingJemallocWithCH.html | 2 +-
.../developers/VeloxDynamicSizingOffheap.html | 2 +-
output/docs/v1.5.0/developers/VeloxUDF.html | 2 +-
.../developers/clickhouse-backend-debug.html | 2 +-
.../v1.5.0/developers/improvement-proposals.html | 2 +-
.../docs/v1.5.0/developers/velox-backend-CI.html | 2 +-
.../developers/velox-backend-build-in-docker.html | 2 +-
.../velox-function-development-guide.html | 2 +-
output/docs/v1.5.0/get-started/ClickHouse.html | 2 +-
output/docs/v1.5.0/get-started/GlutenUI.html | 2 +-
output/docs/v1.5.0/get-started/Velox.html | 2 +-
output/docs/v1.5.0/get-started/VeloxABFS.html | 2 +-
output/docs/v1.5.0/get-started/VeloxGCS.html | 2 +-
output/docs/v1.5.0/get-started/VeloxGPU.html | 2 +-
output/docs/v1.5.0/get-started/VeloxIceberg.html | 2 +-
.../docs/v1.5.0/get-started/VeloxLocalCache.html | 2 +-
output/docs/v1.5.0/get-started/VeloxQAT.html | 2 +-
output/docs/v1.5.0/get-started/VeloxS3.html | 2 +-
.../v1.5.0/get-started/VeloxStageResourceAdj.html | 2 +-
output/docs/v1.5.0/get-started/build-guide.html | 2 +-
.../velox-backend-aggregate-function-support.html | 2 +-
.../velox-backend-generator-function-support.html | 2 +-
.../velox-backend/velox-backend-limitations.html | 2 +-
.../velox-backend-scalar-function-support.html | 2 +-
.../velox-backend-support-progress.html | 2 +-
.../velox-backend-troubleshooting.html | 2 +-
.../velox-backend-window-function-support.html | 2 +-
output/feed.xml | 17 +-
output/index.html | 10 +-
159 files changed, 1224 insertions(+), 1206 deletions(-)
diff --git a/output/blog/index.html b/output/2026/10/09/community-over-code-glasgow-2026.html
similarity index 83%
copy from output/blog/index.html
copy to output/2026/10/09/community-over-code-glasgow-2026.html
index d1706580..920cf25c 100644
--- a/output/blog/index.html
+++ b/output/2026/10/09/community-over-code-glasgow-2026.html
@@ -3,9 +3,9 @@
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>Blog | Apache Gluten</title>
+<title>Community Over Code Glasgow 2026 | Apache Gluten</title>
<meta name="description" content="Gluten is a middle layer responsible for offloading JVM-based SQL engines' execution to native engines.">
-<meta property="og:title" content="Blog | Apache Gluten">
+<meta property="og:title" content="Community Over Code Glasgow 2026 | Apache Gluten">
<meta property="og:description" content="Gluten is a middle layer responsible for offloading JVM-based SQL engines' execution to native engines.">
<meta property="og:type" content="website">
<script>
@@ -84,45 +84,26 @@
<div class="page-wrapper">
<div class="container">
<div class="page-header">
- <h1>Blog</h1>
+ <h1>Community Over Code Glasgow 2026</h1>
+ <p class="post-meta">October 9, 2026 · Apache Gluten Community</p>
</div>
<div class="page-content">
- <ul class="blog-list">
-
- <li>
- <h2><a href="/2025/12/03/GlutenCon2025-poster.html">Glutencon2025 Poster</a></h2>
- <p class="post-meta">December 3, 2025</p>
- <div align="center">
- <img width="800" alt="image" src="/assets/images/GlutenCon2025/poster.png" />
-</div>
-
- </li>
-
- <li>
- <h2><a href="/2025/11/06/glutencon2025-English.html">Glutencon2025 English</a></h2>
- <p class="post-meta">November 6, 2025</p>
- <h1 id="-glutencon-2025--first-annual-conference-officially-launched">🎉 GlutenCon 2025 — First Annual Conference Officially Launched!</h1>
+ <h1 id="community-over-code-glasgow-2026">Community Over Code Glasgow 2026</h1>
+<p>Community over Code Glasgow is less than a month away! <strong>Oct 11–14</strong>, the Apache community gathers in Glasgow, Scotland.</p>
- </li>
-
- <li>
- <h2><a href="/2025/11/06/GlutenCon2025-Chinese.html">Glutencon2025 Chinese</a></h2>
- <p class="post-meta">November 6, 2025</p>
- <h1 id="-glutencon-2025-首届大会正式启动">🎉 GlutenCon 2025 首届大会正式启动！</h1>
+<p>25+ projects are signed up for the hackathon, keynotes are lined up, and the schedule is packed across <strong>20 tracks</strong>. Registration is still open but time is running out — book your travel now. This is where the people behind the projects you depend on come together.</p>
+<p>Apache Gluten will be represented with <strong>2 sessions</strong> at the conference — come find us there!</p>
- </li>
-
- <li>
- <h2><a href="/2024/09/03/Gluten-1.2.0-release.html">Gluten 1.2.0 Release</a></h2>
- <p class="post-meta">September 3, 2024 · Weiting Chen</p>
- <p>On Sept. 3rd 2024, I am thrilled to share that Apache Gluten (incubating) has celebrated its 1st Apache official release, version 1.2.0, featuring the following highlights.</p>
+<ul>
+ <li><a href="https://web.cvent.com/event/ac71ce47-2b5f-424c-abfe-5b48255315fb/summary?RefId=XSummary&session=31f3bd3d-f0e0-4098-be49-022515872f63">Latest Apache Gluten Optimizations: From Planning to Execution</a></li>
+ <li><a href="https://web.cvent.com/event/ac71ce47-2b5f-424c-abfe-5b48255315fb/summary?RefId=XSummary&session=97bc8b56-542c-4ddb-ae5d-11003b3a4717">Apache Gluten: From Incubating to Top-Level Project — Architecture, Ecosystem, and Latest Advances</a></li>
+</ul>
+<p>🔗 <a href="https://communityovercode.apache.org/events/glasgow-2026/">https://communityovercode.apache.org/events/glasgow-2026/</a></p>
- </li>
-
-</ul>
+<p>#CommunityOverCode #ASF2026Glasgow #OpenSource</p>
</div>
</div>
diff --git a/output/archives/index.html b/output/archives/index.html
index 038536d1..c4e746bc 100644
--- a/output/archives/index.html
+++ b/output/archives/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li a { background-image: none; } .site-nav > ul:not [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li a { background-image: none; } .site-nav > ul:not [...]
diff --git a/output/archives/v1.1.1/developers/cpp_code_style/index.html b/output/archives/v1.1.1/developers/cpp_code_style/index.html
index 6f4204e4..bd3982a6 100644
--- a/output/archives/v1.1.1/developers/cpp_code_style/index.html
+++ b/output/archives/v1.1.1/developers/cpp_code_style/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/developers/docker_centos7/index.html b/output/archives/v1.1.1/developers/docker_centos7/index.html
index 1166d2e1..acd508b3 100644
--- a/output/archives/v1.1.1/developers/docker_centos7/index.html
+++ b/output/archives/v1.1.1/developers/docker_centos7/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run -itd --name gluten centos:7 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum -y install epel-release centos-release-scl
diff --git a/output/archives/v1.1.1/developers/docker_centos8.html b/output/archives/v1.1.1/developers/docker_centos8.html
index d082130c..b11ca7f3 100644
--- a/output/archives/v1.1.1/developers/docker_centos8.html
+++ b/output/archives/v1.1.1/developers/docker_centos8.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run -itd --name gluten centos:8 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>#update mirror
diff --git a/output/archives/v1.1.1/developers/docker_ubuntu/index.html b/output/archives/v1.1.1/developers/docker_ubuntu/index.html
index ac3e5d7a..2099a7a9 100644
--- a/output/archives/v1.1.1/developers/docker_ubuntu/index.html
+++ b/output/archives/v1.1.1/developers/docker_ubuntu/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
diff --git a/output/archives/v1.1.1/developers/howto/index.html b/output/archives/v1.1.1/developers/howto/index.html
index 321797fc..1cf162f5 100644
--- a/output/archives/v1.1.1/developers/howto/index.html
+++ b/output/archives/v1.1.1/developers/howto/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-plaintext [...]
mvn clean package -Pspark-3.2 -Pbackends-velox -Prss
mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-velox -am -DtagsToInclude="io.glutenproject.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Darrow.version=11.0.0-gluten -Dexec.skip
diff --git a/output/archives/v1.1.1/developers/howtorelease/index.html b/output/archives/v1.1.1/developers/howtorelease/index.html
index 0a2c3bcb..58e09f98 100644
--- a/output/archives/v1.1.1/developers/howtorelease/index.html
+++ b/output/archives/v1.1.1/developers/howtorelease/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
$ gpg --full-generate-key
# list the GPG keys
diff --git a/output/archives/v1.1.1/developers/index.html b/output/archives/v1.1.1/developers/index.html
index ff6ce183..cea8fdad 100644
--- a/output/archives/v1.1.1/developers/index.html
+++ b/output/archives/v1.1.1/developers/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/developers/microbenchmark/index.html b/output/archives/v1.1.1/developers/microbenchmark/index.html
index c26bc364..415060bd 100644
--- a/output/archives/v1.1.1/developers/microbenchmark/index.html
+++ b/output/archives/v1.1.1/developers/microbenchmark/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3</span>
diff --git a/output/archives/v1.1.1/developers/newto/index.html b/output/archives/v1.1.1/developers/newto/index.html
index 67d452d2..068bd260 100644
--- a/output/archives/v1.1.1/developers/newto/index.html
+++ b/output/archives/v1.1.1/developers/newto/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/archives/v1.1.1/developers/substrait/index.html b/output/archives/v1.1.1/developers/substrait/index.html
index 452762e3..d1b42815 100644
--- a/output/archives/v1.1.1/developers/substrait/index.html
+++ b/output/archives/v1.1.1/developers/substrait/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/developers/velox_function_development/index.html b/output/archives/v1.1.1/developers/velox_function_development/index.html
index 0f73c36a..968c0f3a 100644
--- a/output/archives/v1.1.1/developers/velox_function_development/index.html
+++ b/output/archives/v1.1.1/developers/velox_function_development/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html b/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html
index a53c0fe7..f320de1b 100644
--- a/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html
+++ b/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <p>If you don’t care about development environment, you can skip this part.</p> <p>Otherwise, do:</p> <ol> <li>cl [...]
git clone --recursive --shallow-submodules -b clickhouse_backend https://github.com/Kyligence/ClickHouse.git
</code></pre></div> </div> </li> <li> <p>Configure cpp-ch ${GLUTEN_SOURCE}/cpp-ch can be treated as an add-on of Kyligence/Clickhouse</p> <p>First, initialize some configuration for this add-on:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="nb">export </span><span class="nv">GLUTEN_SOURCE</span><span class="o">=</span>/path/to/gluten
diff --git a/output/archives/v1.1.1/docs/clickhouse/index.html b/output/archives/v1.1.1/docs/clickhouse/index.html
index 14aa917d..ac5f7107 100644
--- a/output/archives/v1.1.1/docs/clickhouse/index.html
+++ b/output/archives/v1.1.1/docs/clickhouse/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/index.html b/output/archives/v1.1.1/docs/index.html
index 7087e5a5..935baa91 100644
--- a/output/archives/v1.1.1/docs/index.html
+++ b/output/archives/v1.1.1/docs/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/velox/build_parameters/index.html b/output/archives/v1.1.1/docs/velox/build_parameters/index.html
index f907f5da..6858b950 100644
--- a/output/archives/v1.1.1/docs/velox/build_parameters/index.html
+++ b/output/archives/v1.1.1/docs/velox/build_parameters/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/velox/gcs/index.html b/output/archives/v1.1.1/docs/velox/gcs/index.html
index 0654286b..45f48ab1 100644
--- a/output/archives/v1.1.1/docs/velox/gcs/index.html
+++ b/output/archives/v1.1.1/docs/velox/gcs/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-sh highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage.roo [...]
</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. All other marks mentioned may b [...]
diff --git a/output/archives/v1.1.1/docs/velox/getting-started/index.html b/output/archives/v1.1.1/docs/velox/getting-started/index.html
index bddf77a6..541a8847 100644
--- a/output/archives/v1.1.1/docs/velox/getting-started/index.html
+++ b/output/archives/v1.1.1/docs/velox/getting-started/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
diff --git a/output/archives/v1.1.1/docs/velox/index.html b/output/archives/v1.1.1/docs/velox/index.html
index fcdfc2e2..6cad481d 100644
--- a/output/archives/v1.1.1/docs/velox/index.html
+++ b/output/archives/v1.1.1/docs/velox/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/velox/s3/index.html b/output/archives/v1.1.1/docs/velox/s3/index.html
index 94bd1254..a5c719bb 100644
--- a/output/archives/v1.1.1/docs/velox/s3/index.html
+++ b/output/archives/v1.1.1/docs/velox/s3/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
diff --git a/output/archives/v1.1.1/index.html b/output/archives/v1.1.1/index.html
index f81672e2..412c49b1 100644
--- a/output/archives/v1.1.1/index.html
+++ b/output/archives/v1.1.1/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.2.1/docs/configuration/index.html b/output/archives/v1.2.1/docs/configuration/index.html
index a7ceb310..de3791a1 100644
--- a/output/archives/v1.2.1/docs/configuration/index.html
+++ b/output/archives/v1.2.1/docs/configuration/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nv">sparkContext</span><span class="o">.</span><span class="py">setLocalProperty</span><span class="o">(</span><span class="n">key</span><span class="o">,</span> <span class="n">value</span><span class="o">)</span>
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"select * from demo_tables"</span><span class="o">).</span><span class="py">show</span><span class="o">()</span>
</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. All other marks mentioned may b [...]
diff --git a/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html b/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html
index 7f5a2f7c..00e3c4bb 100644
--- a/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html
+++ b/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/developers/docker-centos7/index.html b/output/archives/v1.2.1/docs/developers/docker-centos7/index.html
index fff75443..c6094046 100644
--- a/output/archives/v1.2.1/docs/developers/docker-centos7/index.html
+++ b/output/archives/v1.2.1/docs/developers/docker-centos7/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:7 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum <span class="nt">-y</span> <span class="nb">install </span>epel-release centos-release-scl
diff --git a/output/archives/v1.2.1/docs/developers/docker-centos8/index.html b/output/archives/v1.2.1/docs/developers/docker-centos8/index.html
index 2049728f..171770f1 100644
--- a/output/archives/v1.2.1/docs/developers/docker-centos8/index.html
+++ b/output/archives/v1.2.1/docs/developers/docker-centos8/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:8 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">#update mirror</span>
diff --git a/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html b/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html
index 473473cf..80811cbc 100644
--- a/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html
+++ b/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--network</span> host <span class="nt">--name</span> gluten ubuntu:22.04 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
diff --git a/output/archives/v1.2.1/docs/developers/how-to-release/index.html b/output/archives/v1.2.1/docs/developers/how-to-release/index.html
index 6a8b6ee8..10fe033d 100644
--- a/output/archives/v1.2.1/docs/developers/how-to-release/index.html
+++ b/output/archives/v1.2.1/docs/developers/how-to-release/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nv">$ </span>gpg <span class="nt">--full-generate-key</span>
<span class="c"># list the GPG keys</span>
diff --git a/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html b/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html
index 72ac0460..950d5a8a 100644
--- a/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html
+++ b/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <ol> <li>Open the report file and check if there is any vulnerability issue highlighted. We must guarantee all the vulnerability issue has been solved before an official release.</li> </ol> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, an [...]
diff --git a/output/archives/v1.2.1/docs/developers/how-to-use/index.html b/output/archives/v1.2.1/docs/developers/how-to-use/index.html
index b78b2e35..1cc4fac2 100644
--- a/output/archives/v1.2.1/docs/developers/how-to-use/index.html
+++ b/output/archives/v1.2.1/docs/developers/how-to-use/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-bash highl [...]
mvn clean package <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span>
mvn <span class="nb">test</span> <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span> <span class="nt">-pl</span> backends-velox <span class="nt">-am</span> <span class="nt">-DtagsToInclude</span><span class="o">=</span><span class="s2">"io.glutenproject.tags.GenerateExample"</span> <span class="nt">-Dtest</span><span class="o">=</span>none <span class="nt">-DfailIfNoTests</span><span class="o">=</span><span class="nb">false</span> <s [...]
diff --git a/output/archives/v1.2.1/docs/developers/index.html b/output/archives/v1.2.1/docs/developers/index.html
index 399ad36b..bfaed258 100644
--- a/output/archives/v1.2.1/docs/developers/index.html
+++ b/output/archives/v1.2.1/docs/developers/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html b/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html
index 8eb140b1..10e98c15 100644
--- a/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html
+++ b/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3</span>
diff --git a/output/archives/v1.2.1/docs/developers/new-to/index.html b/output/archives/v1.2.1/docs/developers/new-to/index.html
index 565fd715..0048ce5e 100644
--- a/output/archives/v1.2.1/docs/developers/new-to/index.html
+++ b/output/archives/v1.2.1/docs/developers/new-to/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/archives/v1.2.1/docs/developers/substrait/index.html b/output/archives/v1.2.1/docs/developers/substrait/index.html
index d428bf0c..17631265 100644
--- a/output/archives/v1.2.1/docs/developers/substrait/index.html
+++ b/output/archives/v1.2.1/docs/developers/substrait/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html b/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html
index b619b185..0f692848 100644
--- a/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html
+++ b/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/archives/v1.2.1/docs/getting-started/abfs/index.html b/output/archives/v1.2.1/docs/getting-started/abfs/index.html
index 41c68d73..d2aa8175 100644
--- a/output/archives/v1.2.1/docs/getting-started/abfs/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/abfs/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/build-guide/index.html b/output/archives/v1.2.1/docs/getting-started/build-guide/index.html
index 15ae3ca8..55715a66 100644
--- a/output/archives/v1.2.1/docs/getting-started/build-guide/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/build-guide/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html b/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html
index 3881700e..bedec442 100644
--- a/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <p>If you don’t care about development environment, you can skip this part.</p> <p>Otherwise, do:</p> <ol> <li>cl [...]
git clone <span class="nt">--recursive</span> <span class="nt">--shallow-submodules</span> <span class="nt">-b</span> clickhouse_backend https://github.com/Kyligence/ClickHouse.git
</code></pre></div> </div> </li> <li> <p>Configure cpp-ch ${GLUTEN_SOURCE}/cpp-ch can be treated as an add-on of Kyligence/Clickhouse</p> <p>First, initialize some configuration for this add-on:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="nb">export </span><span class="nv">GLUTEN_SOURCE</span><span class="o">=</span>/path/to/gluten
diff --git a/output/archives/v1.2.1/docs/getting-started/gcs/index.html b/output/archives/v1.2.1/docs/getting-started/gcs/index.html
index a44b3ed0..9a96a555 100644
--- a/output/archives/v1.2.1/docs/getting-started/gcs/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/gcs/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage. [...]
</code></pre></div></div> <h2 id="configuring-gcs-max-retry-count"> <a href="#configuring-gcs-max-retry-count" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-count"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry count </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected.</p> <div class="language-shell highligh [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/index.html b/output/archives/v1.2.1/docs/getting-started/index.html
index 86976214..b72c5ff1 100644
--- a/output/archives/v1.2.1/docs/getting-started/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/localcache/index.html b/output/archives/v1.2.1/docs/getting-started/localcache/index.html
index f9b4b988..b1047540 100644
--- a/output/archives/v1.2.1/docs/getting-started/localcache/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/localcache/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.gluten.sql.columnar.backend.velox.memCacheSize // the total size of <span class="k">in</span><span class="nt">-mem</span> cache, default is 128MB.
spark.gluten.sql.columnar.backend.velox.ssdCachePath // the folder to store the cache files, default is <span class="s2">"/tmp"</span><span class="nb">.</span>
spark.gluten.sql.columnar.backend.velox.ssdCacheSize // the total size of the SSD cache, default is 128MB. Velox will <span class="k">do in</span><span class="nt">-mem</span> cache only <span class="k">if </span>this value is 0.
diff --git a/output/archives/v1.2.1/docs/getting-started/s3/index.html b/output/archives/v1.2.1/docs/getting-started/s3/index.html
index 41d7691f..ce61b5bc 100644
--- a/output/archives/v1.2.1/docs/getting-started/s3/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/s3/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
diff --git a/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html b/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html
index a8797f23..0232f096 100644
--- a/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
diff --git a/output/archives/v1.2.1/docs/index.html b/output/archives/v1.2.1/docs/index.html
index 2628a64f..892411f6 100644
--- a/output/archives/v1.2.1/docs/index.html
+++ b/output/archives/v1.2.1/docs/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/index.html b/output/archives/v1.2.1/docs/velox-backend/index.html
index d0b0ea37..303ae1b2 100644
--- a/output/archives/v1.2.1/docs/velox-backend/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/limitations/index.html b/output/archives/v1.2.1/docs/velox-backend/limitations/index.html
index 3b2807c9..c91c90e1 100644
--- a/output/archives/v1.2.1/docs/velox-backend/limitations/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/limitations/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="filesource-format"> <a href="#filesource-format" class="anchor-heading" aria-labelledby="filesource-format"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> FileSource format </h4> <p>Currently, Gluten only fully supports parquet file format and partially support ORC. If other format is used, scan operator falls back to vanilla spark.</p> <h4 id="partitioned-table-scan"> <a href="#partitioned-table-scan" class="anc [...]
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e"</span><span class="o">)</span>
</code></pre></div></div> <p>Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><spa [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/support/index.html b/output/archives/v1.2.1/docs/velox-backend/support/index.html
index a3a5f0c3..4fc78fe6 100644
--- a/output/archives/v1.2.1/docs/velox-backend/support/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/support/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html b/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html
index 732f6ee6..67bbc6be 100644
--- a/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
# velox backend only
rm -rf $SPARK_HOME/jars/flatbuffers-*
rm -rf $SPARK_HOME/jars/arrow-*
diff --git a/output/archives/v1.2.1/docs/velox-backend/udf/index.html b/output/archives/v1.2.1/docs/velox-backend/udf/index.html
index 5e3017c4..72c6d728 100644
--- a/output/archives/v1.2.1/docs/velox-backend/udf/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/udf/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
#include <velox/expression/VectorFunction.h>
#include <velox/udf/Udf.h>
diff --git a/output/archives/v1.2.1/index.html b/output/archives/v1.2.1/index.html
index 15ba5914..f53098a3 100644
--- a/output/archives/v1.2.1/index.html
+++ b/output/archives/v1.2.1/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(4)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(4)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.3.0/configuration.html b/output/archives/v1.3.0/configuration.html
index 9e566690..684f2bdf 100644
--- a/output/archives/v1.3.0/configuration.html
+++ b/output/archives/v1.3.0/configuration.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nv">sparkContext</span><span class="o">.</span><span class="py">setLocalProperty</span><span class="o">(</span><span class="n">key</span><span class="o">,</span> <span class="n">value</span><span class="o">)</span>
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"select * from demo_tables"</span><span class="o">).</span><span class="py">show</span><span class="o">()</span>
</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. All other marks mentioned may b [...]
diff --git a/output/archives/v1.3.0/developers.html b/output/archives/v1.3.0/developers.html
index 75dbd05b..b15b932c 100644
--- a/output/archives/v1.3.0/developers.html
+++ b/output/archives/v1.3.0/developers.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/developers/cpp-code-style/index.html b/output/archives/v1.3.0/developers/cpp-code-style/index.html
index 34db9fe6..582b21ea 100644
--- a/output/archives/v1.3.0/developers/cpp-code-style/index.html
+++ b/output/archives/v1.3.0/developers/cpp-code-style/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/developers/docker-centos7/index.html b/output/archives/v1.3.0/developers/docker-centos7/index.html
index 869ba05d..f97632d9 100644
--- a/output/archives/v1.3.0/developers/docker-centos7/index.html
+++ b/output/archives/v1.3.0/developers/docker-centos7/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:7 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum <span class="nt">-y</span> <span class="nb">install </span>epel-release centos-release-scl
diff --git a/output/archives/v1.3.0/developers/docker-centos8/index.html b/output/archives/v1.3.0/developers/docker-centos8/index.html
index 3d6c9f28..47ccc62d 100644
--- a/output/archives/v1.3.0/developers/docker-centos8/index.html
+++ b/output/archives/v1.3.0/developers/docker-centos8/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:8 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">#update mirror</span>
diff --git a/output/archives/v1.3.0/developers/docker-ubuntu/index.html b/output/archives/v1.3.0/developers/docker-ubuntu/index.html
index fdf27392..7b424b39 100644
--- a/output/archives/v1.3.0/developers/docker-ubuntu/index.html
+++ b/output/archives/v1.3.0/developers/docker-ubuntu/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--network</span> host <span class="nt">--name</span> gluten ubuntu:22.04 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
diff --git a/output/archives/v1.3.0/developers/how-to-security-scan/index.html b/output/archives/v1.3.0/developers/how-to-security-scan/index.html
index 246d98b0..59666a84 100644
--- a/output/archives/v1.3.0/developers/how-to-security-scan/index.html
+++ b/output/archives/v1.3.0/developers/how-to-security-scan/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <ol> <li>Open the report file and check if there is any vulnerability issue highlighted. We must guarantee all the vulnerability issue has been solved before an official release.</li> </ol> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, an [...]
diff --git a/output/archives/v1.3.0/developers/how-to-use/index.html b/output/archives/v1.3.0/developers/how-to-use/index.html
index 8e4730ed..e35ab344 100644
--- a/output/archives/v1.3.0/developers/how-to-use/index.html
+++ b/output/archives/v1.3.0/developers/how-to-use/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-bash highl [...]
mvn clean package <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span>
mvn <span class="nb">test</span> <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span> <span class="nt">-pl</span> backends-velox <span class="nt">-am</span> <span class="nt">-DtagsToInclude</span><span class="o">=</span><span class="s2">"io.glutenproject.tags.GenerateExample"</span> <span class="nt">-Dtest</span><span class="o">=</span>none <span class="nt">-DfailIfNoTests</span><span class="o">=</span><span class="nb">false</span> <s [...]
diff --git a/output/archives/v1.3.0/developers/microbenchmarks/index.html b/output/archives/v1.3.0/developers/microbenchmarks/index.html
index 1e8a59b5..31cc76c8 100644
--- a/output/archives/v1.3.0/developers/microbenchmarks/index.html
+++ b/output/archives/v1.3.0/developers/microbenchmarks/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3</span>
diff --git a/output/archives/v1.3.0/developers/new-to/index.html b/output/archives/v1.3.0/developers/new-to/index.html
index 2f0e1e96..254d8be4 100644
--- a/output/archives/v1.3.0/developers/new-to/index.html
+++ b/output/archives/v1.3.0/developers/new-to/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/archives/v1.3.0/developers/substrait/index.html b/output/archives/v1.3.0/developers/substrait/index.html
index a59ccacf..a1734fdf 100644
--- a/output/archives/v1.3.0/developers/substrait/index.html
+++ b/output/archives/v1.3.0/developers/substrait/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/developers/velox-function-dev/index.html b/output/archives/v1.3.0/developers/velox-function-dev/index.html
index 8005530f..8c4a959f 100644
--- a/output/archives/v1.3.0/developers/velox-function-dev/index.html
+++ b/output/archives/v1.3.0/developers/velox-function-dev/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/archives/v1.3.0/getting-started.html b/output/archives/v1.3.0/getting-started.html
index 58b85e76..ab1fdb06 100644
--- a/output/archives/v1.3.0/getting-started.html
+++ b/output/archives/v1.3.0/getting-started.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/getting-started/abfs/index.html b/output/archives/v1.3.0/getting-started/abfs/index.html
index 5ba097bf..f854deae 100644
--- a/output/archives/v1.3.0/getting-started/abfs/index.html
+++ b/output/archives/v1.3.0/getting-started/abfs/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
diff --git a/output/archives/v1.3.0/getting-started/build-guide/index.html b/output/archives/v1.3.0/getting-started/build-guide/index.html
index 8297e4de..a996b31c 100644
--- a/output/archives/v1.3.0/getting-started/build-guide/index.html
+++ b/output/archives/v1.3.0/getting-started/build-guide/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html b/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html
index 1a61d2d0..dd64a53b 100644
--- a/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html
+++ b/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <p>If you don’t care about development environment, you can skip this part.</p> <p>Otherwise, do:</p> <ol> <li>cl [...]
git clone <span class="nt">--recursive</span> <span class="nt">--shallow-submodules</span> <span class="nt">-b</span> clickhouse_backend https://github.com/Kyligence/ClickHouse.git
</code></pre></div> </div> </li> <li> <p>Configure cpp-ch ${GLUTEN_SOURCE}/cpp-ch can be treated as an add-on of Kyligence/Clickhouse</p> <p>First, initialize some configuration for this add-on:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="nb">export </span><span class="nv">GLUTEN_SOURCE</span><span class="o">=</span>/path/to/gluten
diff --git a/output/archives/v1.3.0/getting-started/gcs/index.html b/output/archives/v1.3.0/getting-started/gcs/index.html
index 64762d72..8d245727 100644
--- a/output/archives/v1.3.0/getting-started/gcs/index.html
+++ b/output/archives/v1.3.0/getting-started/gcs/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage. [...]
</code></pre></div></div> <h2 id="configuring-gcs-max-retry-count"> <a href="#configuring-gcs-max-retry-count" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-count"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry count </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected.</p> <div class="language-shell highligh [...]
diff --git a/output/archives/v1.3.0/getting-started/localcache/index.html b/output/archives/v1.3.0/getting-started/localcache/index.html
index 29c50b8d..038513e8 100644
--- a/output/archives/v1.3.0/getting-started/localcache/index.html
+++ b/output/archives/v1.3.0/getting-started/localcache/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.gluten.sql.columnar.backend.velox.memCacheSize // the total size of <span class="k">in</span><span class="nt">-mem</span> cache, default is 128MB.
spark.gluten.sql.columnar.backend.velox.ssdCachePath // the folder to store the cache files, default is <span class="s2">"/tmp"</span><span class="nb">.</span>
spark.gluten.sql.columnar.backend.velox.ssdCacheSize // the total size of the SSD cache, default is 128MB. Velox will <span class="k">do in</span><span class="nt">-mem</span> cache only <span class="k">if </span>this value is 0.
diff --git a/output/archives/v1.3.0/getting-started/s3/index.html b/output/archives/v1.3.0/getting-started/s3/index.html
index 9733dded..49a296c7 100644
--- a/output/archives/v1.3.0/getting-started/s3/index.html
+++ b/output/archives/v1.3.0/getting-started/s3/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
diff --git a/output/archives/v1.3.0/getting-started/velox-backend/index.html b/output/archives/v1.3.0/getting-started/velox-backend/index.html
index 7e12d06e..ed0f4989 100644
--- a/output/archives/v1.3.0/getting-started/velox-backend/index.html
+++ b/output/archives/v1.3.0/getting-started/velox-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
diff --git a/output/archives/v1.3.0/index.html b/output/archives/v1.3.0/index.html
index 3fbbdaa2..3a5577af 100644
--- a/output/archives/v1.3.0/index.html
+++ b/output/archives/v1.3.0/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(5)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(5)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.3.0/velox-backend.html b/output/archives/v1.3.0/velox-backend.html
index 256d67ac..9de6a4c3 100644
--- a/output/archives/v1.3.0/velox-backend.html
+++ b/output/archives/v1.3.0/velox-backend.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/velox-backend/limitations/index.html b/output/archives/v1.3.0/velox-backend/limitations/index.html
index 73ec1953..0e139c74 100644
--- a/output/archives/v1.3.0/velox-backend/limitations/index.html
+++ b/output/archives/v1.3.0/velox-backend/limitations/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="filesource-format"> <a href="#filesource-format" class="anchor-heading" aria-labelledby="filesource-format"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> FileSource format </h4> <p>Currently, Gluten only fully supports parquet file format and partially support ORC. If other format is used, scan operator falls back to vanilla spark.</p> <h4 id="partitioned-table-scan"> <a href="#partitioned-table-scan" class="anc [...]
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e"</span><span class="o">)</span>
</code></pre></div></div> <p>Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><spa [...]
diff --git a/output/archives/v1.3.0/velox-backend/support/index.html b/output/archives/v1.3.0/velox-backend/support/index.html
index e3f39fc9..6799e23d 100644
--- a/output/archives/v1.3.0/velox-backend/support/index.html
+++ b/output/archives/v1.3.0/velox-backend/support/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/velox-backend/troubleshooting/index.html b/output/archives/v1.3.0/velox-backend/troubleshooting/index.html
index 581aa1b1..f0231b4a 100644
--- a/output/archives/v1.3.0/velox-backend/troubleshooting/index.html
+++ b/output/archives/v1.3.0/velox-backend/troubleshooting/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
# velox backend only
rm -rf $SPARK_HOME/jars/flatbuffers-*
rm -rf $SPARK_HOME/jars/arrow-*
diff --git a/output/archives/v1.3.0/velox-backend/udf/index.html b/output/archives/v1.3.0/velox-backend/udf/index.html
index 873f19b5..87f2c1b7 100644
--- a/output/archives/v1.3.0/velox-backend/udf/index.html
+++ b/output/archives/v1.3.0/velox-backend/udf/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
#include <velox/expression/VectorFunction.h>
#include <velox/udf/Udf.h>
diff --git a/output/archives/v1.4.0/developers.html b/output/archives/v1.4.0/developers.html
index ef58147f..43c4bce1 100644
--- a/output/archives/v1.4.0/developers.html
+++ b/output/archives/v1.4.0/developers.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.4.0/getting-started.html b/output/archives/v1.4.0/getting-started.html
index a31fb942..38952aaa 100644
--- a/output/archives/v1.4.0/getting-started.html
+++ b/output/archives/v1.4.0/getting-started.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.4.0/index.html b/output/archives/v1.4.0/index.html
index 8aa6f391..992941fa 100644
--- a/output/archives/v1.4.0/index.html
+++ b/output/archives/v1.4.0/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(3)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(3)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.4.0/velox-backend.html b/output/archives/v1.4.0/velox-backend.html
index 26591ff4..0f42f891 100644
--- a/output/archives/v1.4.0/velox-backend.html
+++ b/output/archives/v1.4.0/velox-backend.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.5.0/developers.html b/output/archives/v1.5.0/developers.html
index ac64eec8..42acac68 100644
--- a/output/archives/v1.5.0/developers.html
+++ b/output/archives/v1.5.0/developers.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.5.0/getting-started.html b/output/archives/v1.5.0/getting-started.html
index e7287cbe..4613e3fc 100644
--- a/output/archives/v1.5.0/getting-started.html
+++ b/output/archives/v1.5.0/getting-started.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.5.0/index.html b/output/archives/v1.5.0/index.html
index 807b0149..27e12ec4 100644
--- a/output/archives/v1.5.0/index.html
+++ b/output/archives/v1.5.0/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(1)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(1)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.5.0/velox-backend.html b/output/archives/v1.5.0/velox-backend.html
index a541ad8f..883995f1 100644
--- a/output/archives/v1.5.0/velox-backend.html
+++ b/output/archives/v1.5.0/velox-backend.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/assets/js/search-data.json b/output/assets/js/search-data.json
index 307a3da6..da554387 100644
--- a/output/assets/js/search-data.json
+++ b/output/assets/js/search-data.json
@@ -1,7088 +1,7102 @@
{"0": {
+ "doc": "Blog",
+ "title": "Community Over Code Glasgow 2026",
+ "content": "October 9, 2026 · Apache Gluten Community . ",
+ "url": "/blog/",
+
+ "relUrl": "/blog/"
+ },"1": {
+ "doc": "Blog",
+ "title": "Community Over Code Glasgow 2026",
+ "content": "| ",
+ "url": "/blog/#community-over-code-glasgow-2026",
+
+ "relUrl": "/blog/#community-over-code-glasgow-2026"
+ },"2": {
"doc": "Blog",
"title": "Glutencon2025 Poster",
"content": "December 3, 2025 . | ",
"url": "/blog/",
"relUrl": "/blog/"
- },"1": {
+ },"3": {
"doc": "Blog",
"title": "Glutencon2025 English",
"content": "November 6, 2025 . ",
"url": "/blog/",
"relUrl": "/blog/"
- },"2": {
+ },"4": {
"doc": "Blog",
"title": "🎉 GlutenCon 2025 — First Annual Conference Officially Launched!",
"content": "| ",
"url": "/blog/#-glutencon-2025--first-annual-conference-officially-launched",
"relUrl": "/blog/#-glutencon-2025--first-annual-conference-officially-launched"
- },"3": {
+ },"5": {
"doc": "Blog",
"title": "Glutencon2025 Chinese",
"content": "November 6, 2025 . ",
"url": "/blog/",
"relUrl": "/blog/"
- },"4": {
+ },"6": {
"doc": "Blog",
"title": "🎉 GlutenCon 2025 首届大会正式启动！",
"content": "| ",
"url": "/blog/#-glutencon-2025-%E9%A6%96%E5%B1%8A%E5%A4%A7%E4%BC%9A%E6%AD%A3%E5%BC%8F%E5%90%AF%E5%8A%A8",
"relUrl": "/blog/#-glutencon-2025-首届大会正式启动"
- },"5": {
+ },"7": {
"doc": "Blog",
"title": "Gluten 1.2.0 Release",
"content": "September 3, 2024 · Weiting Chen . On Sept. 3rd 2024, I am thrilled to share that Apache Gluten (incubating) has celebrated its 1st Apache official release, version 1.2.0, featuring the following highlights. | . ",
"url": "/blog/",
"relUrl": "/blog/"
- },"6": {
+ },"8": {
"doc": "Blog",
"title": "Blog",
"content": ". | ",
"url": "/blog/",
"relUrl": "/blog/"
- },"7": {
+ },"9": {
"doc": "Community",
"title": "Community",
"content": " ",
"url": "/community/",
"relUrl": "/community/"
- },"8": {
+ },"10": {
"doc": "Community",
"title": "Team",
"content": "The project is governed by a Project Management Committee (PMC). The project’s committers also have write access to the codebase. Mentors . | Name | Apache ID | GitHub ID | . | Felix Cheung | felixcheung | felixcheung | . | Kent Yao | yao | yaooqinn | . | Shao Feng Shi | shaofengshi | shaofengshi | . | Wenli Zhang | ovilia | Ovilia | . | Yu Li | liyu | carp84 | . PMC Members . | Name | Apache ID | GitHub ID | . | Binwei Yang | felixybw | FelixYBW | . | Chang Chen | changc [...]
"url": "/community/#team",
"relUrl": "/community/#team"
- },"9": {
+ },"11": {
"doc": "Community",
"title": "Contact Us",
"content": "This plugin is still under active development now, and doesn’t have a stable release. Welcome to evaluate it. If you encounter any issues or have any suggestions, please submit to our issue list. We’d love to hear your feedback . Apache has meticulously set up dedicated mailing lists for every project, underscoring the pivotal role of these platforms in fostering communication within the Apache community. Mailing lists serve as the backbone for numerous aspects of communi [...]
"url": "/community/#contact-us",
"relUrl": "/community/#contact-us"
- },"10": {
+ },"12": {
"doc": "Contributing to Gluten",
"title": "How to become a committer",
"content": "To initiate your contributions to Gluten, understand the contribution process—any individual can submit patches, documentation, and examples to the project. The PMC routinely incorporates fresh committers from the pool of active contributors, assessing their contributions to Gluten. The criteria for new committers encompass: . | The Scope of Contribution: To expand contributions beyond pull requests, individuals can engage in various activities including reviewing other p [...]
"url": "/contributing/#how-to-become-a-committer",
"relUrl": "/contributing/#how-to-become-a-committer"
- },"11": {
+ },"13": {
"doc": "Contributing to Gluten",
"title": "Contributing to Gluten",
"content": " ",
"url": "/contributing/",
"relUrl": "/contributing/"
- },"12": {
+ },"14": {
"doc": "Downloads",
"title": "Download Apache Gluten",
"content": " ",
"url": "/downloads/#download-apache-gluten",
"relUrl": "/downloads/#download-apache-gluten"
- },"13": {
+ },"15": {
"doc": "Downloads",
"title": "Instructions",
"content": "Select a version of Gluten to download from the list below. We recommend downloading the latest release. Ensure the integrity of the release by verifying the corresponding SHA-512 hashes and signatures with the project release KEYS. You can find verification instructions in the provided guide. ",
"url": "/downloads/#instructions",
"relUrl": "/downloads/#instructions"
- },"14": {
+ },"16": {
"doc": "Downloads",
"title": "The Latest Official Releases",
"content": "Please note that the binary files were compiled on CentOS 7 using static linking. Starting with the 1.5.0 release, the Intel® Xeon® Platinum 8272CL or a similar CPU variant is used (as determined by GitHub for the ubuntu-latest runner). For earlier releases, the Intel® Xeon® Gold 6252 CPU was used. Performance and runtime compatibility are not guaranteed on other operating systems or hardware platforms. For optimal performance and maximum platform compatibility, we recomm [...]
"url": "/downloads/#the-latest-official-releases",
"relUrl": "/downloads/#the-latest-official-releases"
- },"15": {
+ },"17": {
"doc": "Downloads",
"title": "Nightly Releases",
"content": "Development versions of Gluten binaries are also built nightly and hosted at https://nightlies.apache.org/gluten/. Be aware that these binaries are unofficial and recommended for testing purpose only. ",
"url": "/downloads/#nightly-releases",
"relUrl": "/downloads/#nightly-releases"
- },"16": {
+ },"18": {
"doc": "Downloads",
"title": "All Archived Releases",
"content": "| Version | Date | Source Code Release | Binary Release | Release Notes | . | 1.2.1 | Dec. 12 2024 | Source Code (sha512, signature) | Spark 3.2 Binary (sha512, signature) Spark 3.3 Binary (sha512, signature) Spark 3.4 Binary (sha512, signature) Spark 3.5 Binary (sha512, signature) | Notes | . | 1.2.0 | Sept. 3 2024 | Source Code (sha512, signature) | Spark 3.2 Binary (sha512, signature) Spark 3.3 Binary (sha512, signature) Spark 3.4 Binary (sha512, signature) Spark 3.5 B [...]
"url": "/downloads/#all-archived-releases",
"relUrl": "/downloads/#all-archived-releases"
- },"17": {
+ },"19": {
"doc": "Downloads",
"title": "License",
"content": "The software licensed under Apache License 2.0. ",
"url": "/downloads/#license",
"relUrl": "/downloads/#license"
- },"18": {
+ },"20": {
"doc": "Downloads",
"title": "Downloads",
"content": " ",
"url": "/downloads/",
"relUrl": "/downloads/"
- },"19": {
+ },"21": {
"doc": "Home",
"title": "Supercharge SparkSQL with Native Execution",
"content": "Apache Gluten is a middle layer that offloads JVM-based SQL engines to native engines like Velox and ClickHouse — delivering up to 3x+ performance with zero query changes. Get Started View on GitHub Downloads 3.3x TPC-H Speedup 3.0x TPC-DS Speedup 23x Max Single-Query Speedup 15+ Production Adopters ",
"url": "/",
"relUrl": "/"
- },"20": {
+ },"22": {
"doc": "Home",
"title": "Why Gluten?",
"content": "Gluten bridges the gap between Spark's scalability and native engine performance — no query changes required. ⚡ Native Execution . Offloads compute-intensive processing from the JVM to highly optimized native C++ engines for dramatic speedups. 🔌 Drop-in Plugin . Works as a Spark plugin — no changes to your DataFrames, SQL queries, or application code. Just add a JAR. 🔀 Multi-Backend Support . Switch between Velox and ClickHouse backends based on your workload. Extensible [...]
"url": "/",
"relUrl": "/"
- },"21": {
+ },"23": {
"doc": "Home",
"title": "Architecture",
"content": "Spark physical plans are converted to Substrait and executed by native backends, with results returned via the Spark Columnar API. ",
"url": "/",
"relUrl": "/"
- },"22": {
+ },"24": {
"doc": "Home",
"title": "Quick Start",
"content": "Grab a bundle JAR from the downloads page and enable Gluten with a single Spark configuration change. spark-shell \\ --master yarn --deploy-mode client \\ --conf spark.plugins=org.apache.gluten.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --conf spark.shuffle.manager=org.apache.spark.shuffle.sort.ColumnarShuffleManager \\ --jars gluten-velox-bundle-spark3.5_2.12-centos_7_x86_64-1.6.0.jar . Read the full documentation → . ",
"url": "/",
"relUrl": "/"
- },"23": {
+ },"25": {
"doc": "Home",
"title": "Performance",
"content": "Benchmarked on TPC-H and TPC-DS at 3TB scale on Intel® Xeon® Platinum 8592+ with Spark 3.3.1. TPC-H — Overall 3.34x speedup with Gluten + Velox TPC-H — Up to 23.45x speedup on individual queries TPC-DS — Overall 3.02x speedup with Gluten + Velox TPC-DS — Up to 13.75x speedup on individual queries ",
"url": "/",
"relUrl": "/"
- },"24": {
+ },"26": {
"doc": "Home",
"title": "Trusted in Production",
"content": "Used by leading technology companies across the globe. Microsoft Google IBM Alibaba Cloud Bytedance Uber Tencent Meituan Baidu Pinterest Oracle Kyligence BIGO Netease Intel View all adopters → . ",
"url": "/",
"relUrl": "/"
- },"25": {
+ },"27": {
"doc": "Home",
"title": "Latest News",
- "content": "December 3, 2025 Glutencon2025 Poster . November 6, 2025 Glutencon2025 English . November 6, 2025 Glutencon2025 Chinese . All posts → . ",
+ "content": "October 9, 2026 Community Over Code Glasgow 2026 . December 3, 2025 Glutencon2025 Poster . November 6, 2025 Glutencon2025 English . All posts → . ",
"url": "/",
"relUrl": "/"
- },"26": {
+ },"28": {
"doc": "Home",
"title": "Ready to accelerate your Spark workloads?",
"content": "Join the community and start using Gluten today. Read the Docs Join the Community ",
"url": "/",
"relUrl": "/"
- },"27": {
+ },"29": {
"doc": "Home",
"title": "Thanks to Our Contributors",
"content": "Gluten is built by a vibrant open-source community. ",
"url": "/",
"relUrl": "/"
- },"28": {
+ },"30": {
"doc": "Home",
"title": "Home",
"content": "🚀 Apache Top-Level Project ",
"url": "/",
"relUrl": "/"
- },"29": {
+ },"31": {
"doc": "Powered by Gluten",
"title": "Project and Product Name using Apache Gluten",
"content": "Organizations developing products and projects for use with Apache Gluten, as well as related marketing materials, should ensure they respect the trademark of “Apache Gluten” and its logo. For comprehensive and authoritative guidance on the proper usage of ASF trademarks, please refer to the ASF Trademarks Guidance and the associated FAQ. ",
"url": "/poweredby/#project-and-product-name-using-apache-gluten",
"relUrl": "/poweredby/#project-and-product-name-using-apache-gluten"
- },"30": {
+ },"32": {
"doc": "Powered by Gluten",
"title": "Companies and Organizations",
"content": "To add yourself to the list, please email dev@gluten.apache.org with the following information: your organization name, URL, a list of the Gluten components you are using, and a brief description of your use case. Sorted by Company Name . | Company Name | Description | Link | . | Alibaba | Alibaba has chosen Gluten as a part of their public cloud solution. | Link | . | Baidu | Baidu has used gluten in multiple business scenarios. | | . | Bytedance | EMR in Bytedance’s V [...]
"url": "/poweredby/#companies-and-organizations",
"relUrl": "/poweredby/#companies-and-organizations"
- },"31": {
+ },"33": {
"doc": "Powered by Gluten",
"title": "Powered by Gluten",
"content": " ",
"url": "/poweredby/",
"relUrl": "/poweredby/"
- },"32": {
+ },"34": {
"doc": "Gluten References",
"title": "Gluten related Use Cases and Publications",
"content": "For more information on Apache Gluten, including related articles and videos, please refer to the following links. | Title | Company | Publish Date | Reference | . | Gluten: A middle layer to offload Spark SQL to Native | Intel | Jul. 2022 | Link | . | Best Exploration of Columnar Shuffle Design | Intel | Jul. 2022 | Link | . | Gluten Introduction | Intel | Sept. 2022 | Link | . | Introducing Velox | Meta | Mar. 2023 | Link | . | Gluten in Kyligence(in Chinese) | Kyligenc [...]
"url": "/references/#gluten-related-use-cases-and-publications",
"relUrl": "/references/#gluten-related-use-cases-and-publications"
- },"33": {
+ },"35": {
"doc": "Gluten References",
"title": "Media References",
"content": "| Title | Platform | Publication | Author | Reference | . | Accelerating Apache Spark with Gluten & Velox | Medium | | Angel Conde | Link | . | Accelerate Spark SQL queries with Gluten | Medium | Intel Analytics Software | Weiting Chen | Link | . | Apache Spark Native Engine | Medium | Apache Kyuubi | Apache Kyuubi | Link | . | Native Spark Engine on watsonx.data on Cloud Pak for Data (CPD) using CAS feature | Medium | | Mrudula Madiraju | Link | . | Velox Native [...]
"url": "/references/#media-references",
"relUrl": "/references/#media-references"
- },"34": {
+ },"36": {
"doc": "Gluten References",
"title": "Related Documentation",
"content": "| Title | Project | Reference | . | Deploy Gluten with Apache Kyuubi | Apache Kyuubi | Link | . ",
"url": "/references/#related-documentation",
"relUrl": "/references/#related-documentation"
- },"35": {
+ },"37": {
"doc": "Gluten References",
"title": "Gluten References",
"content": " ",
"url": "/references/",
"relUrl": "/references/"
- },"36": {
+ },"38": {
"doc": "Archives",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/#gluten-documents-by-version",
"relUrl": "/archives/#gluten-documents-by-version"
- },"37": {
+ },"39": {
"doc": "Archives",
"title": "Archives",
"content": " ",
"url": "/archives/",
"relUrl": "/archives/"
- },"38": {
+ },"40": {
"doc": "Documentation(Latest)",
"title": "Documentation(Latest)",
"content": " Click here to view the latest documentation . If you are not redirected, click here. ",
"url": "/docs/latest.html",
"relUrl": "/docs/latest.html"
- },"39": {
+ },"41": {
"doc": "v1.1.1/CPP Code Style",
"title": "Gluten CPP Core Guidelines",
"content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#gluten-cpp-core-guidelines",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#gluten-cpp-core-guidelines"
- },"40": {
+ },"42": {
"doc": "v1.1.1/CPP Code Style",
"title": "Philosophy",
"content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#philosophy",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#philosophy"
- },"41": {
+ },"43": {
"doc": "v1.1.1/CPP Code Style",
"title": "Code Formatting",
"content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#code-formatting",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#code-formatting"
- },"42": {
+ },"44": {
"doc": "v1.1.1/CPP Code Style",
"title": "Naming Conventions",
"content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#naming-conventions",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#naming-conventions"
- },"43": {
+ },"45": {
"doc": "v1.1.1/CPP Code Style",
"title": "Designs",
"content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#designs",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#designs"
- },"44": {
+ },"46": {
"doc": "v1.1.1/CPP Code Style",
"title": "Source File & Header File",
"content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#source-file--header-file",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#source-file--header-file"
- },"45": {
+ },"47": {
"doc": "v1.1.1/CPP Code Style",
"title": "Class",
"content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#class",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#class"
- },"46": {
+ },"48": {
"doc": "v1.1.1/CPP Code Style",
"title": "Function",
"content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#function",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#function"
- },"47": {
+ },"49": {
"doc": "v1.1.1/CPP Code Style",
"title": "Variable",
"content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#variable",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#variable"
- },"48": {
+ },"50": {
"doc": "v1.1.1/CPP Code Style",
"title": "Constant",
"content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#constant",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#constant"
- },"49": {
+ },"51": {
"doc": "v1.1.1/CPP Code Style",
"title": "Macro",
"content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#macro",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#macro"
- },"50": {
+ },"52": {
"doc": "v1.1.1/CPP Code Style",
"title": "Namespace",
"content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#namespace",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#namespace"
- },"51": {
+ },"53": {
"doc": "v1.1.1/CPP Code Style",
"title": "Resource Management",
"content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#resource-management",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#resource-management"
- },"52": {
+ },"54": {
"doc": "v1.1.1/CPP Code Style",
"title": "Exception",
"content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#exception",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#exception"
- },"53": {
+ },"55": {
"doc": "v1.1.1/CPP Code Style",
"title": "Code Comment",
"content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#code-comment",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#code-comment"
- },"54": {
+ },"56": {
"doc": "v1.1.1/CPP Code Style",
"title": "References",
"content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#references",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#references"
- },"55": {
+ },"57": {
"doc": "v1.1.1/CPP Code Style",
"title": "v1.1.1/CPP Code Style",
"content": " ",
"url": "/archives/v1.1.1/developers/cpp_code_style/",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/"
- },"56": {
+ },"58": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to understand the key work of Gluten?",
"content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
"url": "/archives/v1.1.1/developers/howto/#how-to-understand-the-key-work-of-gluten",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-understand-the-key-work-of-gluten"
- },"57": {
+ },"59": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to debug in Gluten?",
"content": " ",
"url": "/archives/v1.1.1/developers/howto/#how-to-debug-in-gluten",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-debug-in-gluten"
- },"58": {
+ },"60": {
"doc": "v1.1.1/How To Use Gluten",
"title": "1 How to debug C++",
"content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
"url": "/archives/v1.1.1/developers/howto/#1-how-to-debug-c",
"relUrl": "/archives/v1.1.1/developers/howto/#1-how-to-debug-c"
- },"59": {
+ },"61": {
"doc": "v1.1.1/How To Use Gluten",
"title": "2 How to debug plan validation process",
"content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
"url": "/archives/v1.1.1/developers/howto/#2-how-to-debug-plan-validation-process",
"relUrl": "/archives/v1.1.1/developers/howto/#2-how-to-debug-plan-validation-process"
- },"60": {
+ },"62": {
"doc": "v1.1.1/How To Use Gluten",
"title": "3 How to debug Java/Scala",
"content": "wait to add . ",
"url": "/archives/v1.1.1/developers/howto/#3-how-to-debug-javascala",
"relUrl": "/archives/v1.1.1/developers/howto/#3-how-to-debug-javascala"
- },"61": {
+ },"63": {
"doc": "v1.1.1/How To Use Gluten",
"title": "4 How to debug with core-dump",
"content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
"url": "/archives/v1.1.1/developers/howto/#4-how-to-debug-with-core-dump",
"relUrl": "/archives/v1.1.1/developers/howto/#4-how-to-debug-with-core-dump"
- },"62": {
+ },"64": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to run TPC-H on Velox backend",
"content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox-be by workflow, you can refer to velox_be.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, prepare the d [...]
"url": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-h-on-velox-backend",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-h-on-velox-backend"
- },"63": {
+ },"65": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to run TPC-DS",
"content": "wait to add . ",
"url": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-ds",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-ds"
- },"64": {
+ },"66": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to track the memory exhaust problem",
"content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
"url": "/archives/v1.1.1/developers/howto/#how-to-track-the-memory-exhaust-problem",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-track-the-memory-exhaust-problem"
- },"65": {
+ },"67": {
"doc": "v1.1.1/How To Use Gluten",
"title": "v1.1.1/How To Use Gluten",
"content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
"url": "/archives/v1.1.1/developers/howto/",
"relUrl": "/archives/v1.1.1/developers/howto/"
- },"66": {
+ },"68": {
"doc": "v1.1.1/How To Release",
"title": "How to Release",
"content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/archives/v1.1.1/developers/howtorelease/#how-to-release",
"relUrl": "/archives/v1.1.1/developers/howtorelease/#how-to-release"
- },"67": {
+ },"69": {
"doc": "v1.1.1/How To Release",
"title": "Release Process",
"content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
"url": "/archives/v1.1.1/developers/howtorelease/#release-process",
"relUrl": "/archives/v1.1.1/developers/howtorelease/#release-process"
- },"68": {
+ },"70": {
"doc": "v1.1.1/How To Release",
"title": "v1.1.1/How To Release",
"content": " ",
"url": "/archives/v1.1.1/developers/howtorelease/",
"relUrl": "/archives/v1.1.1/developers/howtorelease/"
- },"69": {
+ },"71": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Generate Micro Benchmarks for Velox Backend",
"content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#generate-micro-benchmarks-for-velox-backend",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#generate-micro-benchmarks-for-velox-backend"
- },"70": {
+ },"72": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Try the example",
"content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#try-the-example",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#try-the-example"
- },"71": {
+ },"73": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Generate Substrait plan and input for any query",
"content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#generate-substrait-plan-and-input-for-any-query",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#generate-substrait-plan-and-input-for-any-query"
- },"72": {
+ },"74": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Save ouput to parquet to analyze",
"content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
"url": "/archives/v1.1.1/developers/microbenchmark/#save-ouput-to-parquet-to-analyze",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#save-ouput-to-parquet-to-analyze"
- },"73": {
+ },"75": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Add shuffle write process",
"content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#add-shuffle-write-process",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#add-shuffle-write-process"
- },"74": {
+ },"76": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Simulate Spark with multiple processes and threads",
"content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#simulate-spark-with-multiple-processes-and-threads",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#simulate-spark-with-multiple-processes-and-threads"
- },"75": {
+ },"77": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "v1.1.1/Micro Benchmarks for Velox Backend",
"content": " ",
"url": "/archives/v1.1.1/developers/microbenchmark/",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/"
- },"76": {
+ },"78": {
"doc": "v1.1.1/New To Gluten",
"title": "Environment",
"content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
"url": "/archives/v1.1.1/developers/newto/#environment",
"relUrl": "/archives/v1.1.1/developers/newto/#environment"
- },"77": {
+ },"79": {
"doc": "v1.1.1/New To Gluten",
"title": "Openjdk8",
"content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
"url": "/archives/v1.1.1/developers/newto/#openjdk8",
"relUrl": "/archives/v1.1.1/developers/newto/#openjdk8"
- },"78": {
+ },"80": {
"doc": "v1.1.1/New To Gluten",
"title": "Maven 3.6.3 or above",
"content": "Maven Dowload Page And then set the environment setting. ",
"url": "/archives/v1.1.1/developers/newto/#maven-363-or-above",
"relUrl": "/archives/v1.1.1/developers/newto/#maven-363-or-above"
- },"79": {
+ },"81": {
"doc": "v1.1.1/New To Gluten",
"title": "GCC 9.4 or above",
"content": " ",
"url": "/archives/v1.1.1/developers/newto/#gcc-94-or-above",
"relUrl": "/archives/v1.1.1/developers/newto/#gcc-94-or-above"
- },"80": {
+ },"82": {
"doc": "v1.1.1/New To Gluten",
"title": "Compile gluten using debug mode",
"content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
"url": "/archives/v1.1.1/developers/newto/#compile-gluten-using-debug-mode",
"relUrl": "/archives/v1.1.1/developers/newto/#compile-gluten-using-debug-mode"
- },"81": {
+ },"83": {
"doc": "v1.1.1/New To Gluten",
"title": "Java/scala code development with Intellij",
"content": " ",
"url": "/archives/v1.1.1/developers/newto/#javascala-code-development-with-intellij",
"relUrl": "/archives/v1.1.1/developers/newto/#javascala-code-development-with-intellij"
- },"82": {
+ },"84": {
"doc": "v1.1.1/New To Gluten",
"title": "Linux intellij local debug",
"content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
"url": "/archives/v1.1.1/developers/newto/#linux-intellij-local-debug",
"relUrl": "/archives/v1.1.1/developers/newto/#linux-intellij-local-debug"
- },"83": {
+ },"85": {
"doc": "v1.1.1/New To Gluten",
"title": "Windows/Mac intellij remote debug",
"content": "If you have Ultimate intellij, you can try to debug remotely. ",
"url": "/archives/v1.1.1/developers/newto/#windowsmac-intellij-remote-debug",
"relUrl": "/archives/v1.1.1/developers/newto/#windowsmac-intellij-remote-debug"
- },"84": {
+ },"86": {
"doc": "v1.1.1/New To Gluten",
"title": "Set up gluten project",
"content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
"url": "/archives/v1.1.1/developers/newto/#set-up-gluten-project",
"relUrl": "/archives/v1.1.1/developers/newto/#set-up-gluten-project"
- },"85": {
+ },"87": {
"doc": "v1.1.1/New To Gluten",
"title": "Java/Scala code style",
"content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
"url": "/archives/v1.1.1/developers/newto/#javascala-code-style",
"relUrl": "/archives/v1.1.1/developers/newto/#javascala-code-style"
- },"86": {
+ },"88": {
"doc": "v1.1.1/New To Gluten",
"title": "CPP code development with Visual Studio Code",
"content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
"url": "/archives/v1.1.1/developers/newto/#cpp-code-development-with-visual-studio-code",
"relUrl": "/archives/v1.1.1/developers/newto/#cpp-code-development-with-visual-studio-code"
- },"87": {
+ },"89": {
"doc": "v1.1.1/New To Gluten",
"title": "Usage",
"content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
"url": "/archives/v1.1.1/developers/newto/#usage",
"relUrl": "/archives/v1.1.1/developers/newto/#usage"
- },"88": {
+ },"90": {
"doc": "v1.1.1/New To Gluten",
"title": "Usefule notes",
"content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
"url": "/archives/v1.1.1/developers/newto/#usefule-notes",
"relUrl": "/archives/v1.1.1/developers/newto/#usefule-notes"
- },"89": {
+ },"91": {
"doc": "v1.1.1/New To Gluten",
"title": "Debug cpp code with coredump",
"content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
"url": "/archives/v1.1.1/developers/newto/#debug-cpp-code-with-coredump",
"relUrl": "/archives/v1.1.1/developers/newto/#debug-cpp-code-with-coredump"
- },"90": {
+ },"92": {
"doc": "v1.1.1/New To Gluten",
"title": "Debug cpp with gdb",
"content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
"url": "/archives/v1.1.1/developers/newto/#debug-cpp-with-gdb",
"relUrl": "/archives/v1.1.1/developers/newto/#debug-cpp-with-gdb"
- },"91": {
+ },"93": {
"doc": "v1.1.1/New To Gluten",
"title": "Run TPC-H and TPC-DS",
"content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_be.yml . ",
"url": "/archives/v1.1.1/developers/newto/#run-tpc-h-and-tpc-ds",
"relUrl": "/archives/v1.1.1/developers/newto/#run-tpc-h-and-tpc-ds"
- },"92": {
+ },"94": {
"doc": "v1.1.1/New To Gluten",
"title": "Run gluten+velox on clean machine",
"content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
"url": "/archives/v1.1.1/developers/newto/#run-glutenvelox-on-clean-machine",
"relUrl": "/archives/v1.1.1/developers/newto/#run-glutenvelox-on-clean-machine"
- },"93": {
+ },"95": {
"doc": "v1.1.1/New To Gluten",
"title": "v1.1.1/New To Gluten",
"content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
"url": "/archives/v1.1.1/developers/newto/",
"relUrl": "/archives/v1.1.1/developers/newto/"
- },"94": {
+ },"96": {
"doc": "v1.1.1/Substrait Modifications",
"title": "Substrait Modifications in Gluten",
"content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
"url": "/archives/v1.1.1/developers/substrait/#substrait-modifications-in-gluten",
"relUrl": "/archives/v1.1.1/developers/substrait/#substrait-modifications-in-gluten"
- },"95": {
+ },"97": {
"doc": "v1.1.1/Substrait Modifications",
"title": "Modifications to algebra.proto",
"content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
"url": "/archives/v1.1.1/developers/substrait/#modifications-to-algebraproto",
"relUrl": "/archives/v1.1.1/developers/substrait/#modifications-to-algebraproto"
- },"96": {
+ },"98": {
"doc": "v1.1.1/Substrait Modifications",
"title": "Modifications to type.proto",
"content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
"url": "/archives/v1.1.1/developers/substrait/#modifications-to-typeproto",
"relUrl": "/archives/v1.1.1/developers/substrait/#modifications-to-typeproto"
- },"97": {
+ },"99": {
"doc": "v1.1.1/Substrait Modifications",
"title": "v1.1.1/Substrait Modifications",
"content": " ",
"url": "/archives/v1.1.1/developers/substrait/",
"relUrl": "/archives/v1.1.1/developers/substrait/"
- },"98": {
+ },"100": {
"doc": "v1.1.1/Docker script for CentOS 7",
"title": "v1.1.1/Docker script for CentOS 7",
"content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ sudo # gluten need maven version >=3.6.3 wget https://downloads.apache.org/m [...]
"url": "/archives/v1.1.1/developers/docker_centos7/",
"relUrl": "/archives/v1.1.1/developers/docker_centos7/"
- },"99": {
+ },"101": {
"doc": "v1.1.1/Docker script for CentOS 8",
"title": "v1.1.1/Docker script for CentOS 8",
"content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
"url": "/archives/v1.1.1/developers/docker_centos8",
"relUrl": "/archives/v1.1.1/developers/docker_centos8"
- },"100": {
+ },"102": {
"doc": "v1.1.1/Docker script for Ubuntu 22.04/20.04",
"title": "v1.1.1/Docker script for Ubuntu 22.04/20.04",
"content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
"url": "/archives/v1.1.1/developers/docker_ubuntu/",
"relUrl": "/archives/v1.1.1/developers/docker_ubuntu/"
- },"101": {
+ },"103": {
"doc": "v1.1.1/Developers",
"title": "Gluten Developers",
"content": "This document provides a developer overview of the project and covers the following topics: . ",
"url": "/archives/v1.1.1/developers/#gluten-developers",
"relUrl": "/archives/v1.1.1/developers/#gluten-developers"
- },"102": {
+ },"104": {
"doc": "v1.1.1/Developers",
"title": "v1.1.1/Developers",
"content": " ",
"url": "/archives/v1.1.1/developers/",
"relUrl": "/archives/v1.1.1/developers/"
- },"103": {
+ },"105": {
"doc": "v1.1.1/Velox Function Development",
"title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
"content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
"url": "/archives/v1.1.1/developers/velox_function_development/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
"relUrl": "/archives/v1.1.1/developers/velox_function_development/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"104": {
+ },"106": {
"doc": "v1.1.1/Velox Function Development",
"title": "v1.1.1/Velox Function Development",
"content": " ",
"url": "/archives/v1.1.1/developers/velox_function_development/",
"relUrl": "/archives/v1.1.1/developers/velox_function_development/"
- },"105": {
+ },"107": {
"doc": "v1.1.1/ClickHouse Backend",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.1.1/docs/clickhouse/#gluten-documents-by-version",
"relUrl": "/archives/v1.1.1/docs/clickhouse/#gluten-documents-by-version"
- },"106": {
+ },"108": {
"doc": "v1.1.1/ClickHouse Backend",
"title": "v1.1.1/ClickHouse Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/clickhouse/",
"relUrl": "/archives/v1.1.1/docs/clickhouse/"
- },"107": {
+ },"109": {
"doc": "v1.1.1/Getting Started with ClickHouse Backend",
"title": "ClickHouse Backend",
"content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
"url": "/archives/v1.1.1/docs/clickhouse/getting-started/#clickhouse-backend",
"relUrl": "/archives/v1.1.1/docs/clickhouse/getting-started/#clickhouse-backend"
- },"108": {
+ },"110": {
"doc": "v1.1.1/Getting Started with ClickHouse Backend",
"title": "v1.1.1/Getting Started with ClickHouse Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/clickhouse/getting-started/",
"relUrl": "/archives/v1.1.1/docs/clickhouse/getting-started/"
- },"109": {
+ },"111": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Supported Version",
"content": "| Type | Version |——-|——————————| Spark | 3.2.2, 3.3.1 | OS | Ubuntu20.04/22.04, Centos7/8 | jdk | openjdk8 | scala | 2.12 . Spark3.4.0 support is still WIP. TPCH/DS can pass, UT is not yet passed. There are pending PRs for jdk11 support. Currently, the mvn script can automatically fetch and build all dependency libraries incluing Velox. Our nightly build still use Velox under oap-project. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#supported-version",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#supported-version"
- },"110": {
+ },"112": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Prerequisite",
"content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Gluten builds with Spark3.2.x and Spark3.3.x now but only fully tested in CI with 3.2.2 and 3.3.1. We will add/update supported/tested versions according to the upstream changes. we need to set up the JAVA_HOME env. Currently, java 8 is required and the support for j [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#prerequisite",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#prerequisite"
- },"111": {
+ },"113": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Build Gluten with Velox Backend",
"content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . cd /path/to/gluten ## The script builds two jars for spark 3.2.2 and 3.3.1./dev/buildbundle-veloxbe.sh ## After a complete build, if you need to re-build the project and only some gluten code is changed, ## you can use the following command to skip building velox and protobuf. [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-velox-backend",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-velox-backend"
- },"112": {
+ },"114": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Dependency library deployment",
"content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#dependency-library-deployment",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#dependency-library-deployment"
- },"113": {
+ },"115": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "HDFS support",
"content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#hdfs-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#hdfs-support"
- },"114": {
+ },"116": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "AWS S3 support",
"content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#aws-s3-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#aws-s3-support"
- },"115": {
+ },"117": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Celeborn support",
"content": "Gluten with velox backend supports Celeborn as remote shuffle service. Below introduction is used to enable this feature . First refer to this URL(https://github.com/apache/incubator-celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable rss profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Prss -DskipTests . Then add the Gluten and Spark Celeborn Client packages to your Spark application’s classpath(usua [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#celeborn-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#celeborn-support"
- },"116": {
+ },"118": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "DeltaLake Support",
"content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with velox backends [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#deltalake-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#deltalake-support"
- },"117": {
+ },"119": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Iceberg Support",
"content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Then, put the additional gluten-iceberg jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-iceberg jar is in gluten-iceberg/target directory. After the two steps, you can query icebe [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#iceberg-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#iceberg-support"
- },"118": {
+ },"120": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Coverage",
"content": "Spark3.3 has 387 functions in total. ~240 are commonly used. Velox’s functions have two category, Presto and Spark. Presto has 124 functions implemented. Spark has 62 functions. Spark functions are verified to have the same result as Vanilla Spark. Some Presto functions have the same result as Vanilla Spark but some others have different. Gluten prefer to use Spark functions firstly. If it’s not in Spark’s list but implemented in Presto, we currently offload to Presto one [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#coverage",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#coverage"
- },"119": {
+ },"121": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Spill (Experimental)",
"content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#spill-experimental",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#spill-experimental"
- },"120": {
+ },"122": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Velox User-Defined Functions (UDF)",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#velox-user-defined-functions-udf",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#velox-user-defined-functions-udf"
- },"121": {
+ },"123": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Introduction",
"content": "Velox backend supports User-Defined Functions (UDF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#introduction",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#introduction"
- },"122": {
+ },"124": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Creating a UDF library",
"content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#creating-a-udf-library",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#creating-a-udf-library"
- },"123": {
+ },"125": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Building the UDF library",
"content": "To build the UDF library, users need to compile the C++ code and link to libvelox.so. It’s recommended to create a CMakeLists.txt for the project. Here’s an example: . project(myudf) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(GLUTEN_HOME /path/to/gluten) add_library(myudf SHARED \"MyUDF.cpp\") find_library(VELOX_LIBRARY REQUIRED NAMES velox HINTS ${GLUTEN_HOME}/cpp/build/releases NO_DEFAULT_PATH) target_include_directories(myudf PRIVATE ${GLUTEN_HO [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#building-the-udf-library",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#building-the-udf-library"
- },"124": {
+ },"126": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Using UDF in Gluten",
"content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the libray paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#using-udf-in-gluten",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#using-udf-in-gluten"
- },"125": {
+ },"127": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Try the example",
"content": "We provided an Velox UDF example file MyUDF.cpp. After building gluten cpp, you can find the example library at /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so . Start spark-shell or spark-sql with below configuration . --files /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so --conf spark.gluten.sql.columnar.backend.velox.udfLibraryPaths=libmyudf.so . Run query. The functions myudf1 and myudf2 increment the input value by a constant of 5 . select myudf1(1 [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#try-the-example",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#try-the-example"
- },"126": {
+ },"128": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "High-Bandwidth Memory (HBM) support",
"content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#high-bandwidth-memory-hbm-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#high-bandwidth-memory-hbm-support"
- },"127": {
+ },"129": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Build Gluten with HBM",
"content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds two jars for spark 3.2.2 and 3.3.1./dev/buildbundle-veloxbe.sh --enable_hbm=ON . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-hbm",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-hbm"
- },"128": {
+ },"130": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Configure and enable HBM in Spark Application",
"content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#configure-and-enable-hbm-in-spark-application",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#configure-and-enable-hbm-in-spark-application"
- },"129": {
+ },"131": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Intel® QuickAssist Technology (QAT) support",
"content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#intel-quickassist-technology-qat-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#intel-quickassist-technology-qat-support"
- },"130": {
+ },"132": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Software Requirements",
"content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#software-requirements",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#software-requirements"
- },"131": {
+ },"133": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Build Gluten with QAT",
"content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-qat",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-qat"
- },"132": {
+ },"134": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Enable QAT with Gzip/Zstd for shuffle compression",
"content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#enable-qat-with-gzipzstd-for-shuffle-compression",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#enable-qat-with-gzipzstd-for-shuffle-compression"
- },"133": {
+ },"135": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "QAT driver references",
"content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#qat-driver-references",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#qat-driver-references"
- },"134": {
+ },"136": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
"content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#intel-in-memory-analytics-accelerator-iaaiax-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#intel-in-memory-analytics-accelerator-iaaiax-support"
- },"135": {
+ },"137": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Build Gluten with IAA",
"content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-iaa",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-iaa"
- },"136": {
+ },"138": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Enable IAA with Gzip Compression for shuffle compression",
"content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#enable-iaa-with-gzip-compression-for-shuffle-compression",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#enable-iaa-with-gzip-compression-for-shuffle-compression"
- },"137": {
+ },"139": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "IAA references",
"content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#iaa-references",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#iaa-references"
- },"138": {
+ },"140": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
"content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
- },"139": {
+ },"141": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Data preparation",
"content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#data-preparation",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#data-preparation"
- },"140": {
+ },"142": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Submit the Spark SQL job",
"content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#submit-the-spark-sql-job",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#submit-the-spark-sql-job"
- },"141": {
+ },"143": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Result",
"content": "wholestagetransformer indicates that the offload works. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#result",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#result"
- },"142": {
+ },"144": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Performance",
"content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#performance",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#performance"
- },"143": {
+ },"145": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "External reference setup",
"content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#external-reference-setup",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#external-reference-setup"
- },"144": {
+ },"146": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Gluten UI",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-ui",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-ui"
- },"145": {
+ },"147": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Gluten event",
"content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-event",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-event"
- },"146": {
+ },"148": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "SQL tab",
"content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#sql-tab",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#sql-tab"
- },"147": {
+ },"149": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "History server",
"content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#history-server",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#history-server"
- },"148": {
+ },"150": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Native plan string",
"content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-string",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-string"
- },"149": {
+ },"151": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Native plan with stats",
"content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-with-stats",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-with-stats"
- },"150": {
+ },"152": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Gluten Implicits",
"content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-implicits",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-implicits"
- },"151": {
+ },"153": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "v1.1.1/Getting-Started with Velox Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/getting-started/",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/"
- },"152": {
+ },"154": {
"doc": "v1.1.1/Velox Backend",
"title": "Gluten Velox Backend Documents",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/#gluten-velox-backend-documents",
"relUrl": "/archives/v1.1.1/docs/velox/#gluten-velox-backend-documents"
- },"153": {
+ },"155": {
"doc": "v1.1.1/Velox Backend",
"title": "v1.1.1/Velox Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/",
"relUrl": "/archives/v1.1.1/docs/velox/"
- },"154": {
+ },"156": {
"doc": "v1.1.1/Build Parameters for Velox Backend",
"title": "Build Parameters",
"content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | ON | . | build_protob [...]
"url": "/archives/v1.1.1/docs/velox/build_parameters/#build-parameters",
"relUrl": "/archives/v1.1.1/docs/velox/build_parameters/#build-parameters"
- },"155": {
+ },"157": {
"doc": "v1.1.1/Build Parameters for Velox Backend",
"title": "Gluten Jar for Deployment",
"content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . ",
"url": "/archives/v1.1.1/docs/velox/build_parameters/#gluten-jar-for-deployment",
"relUrl": "/archives/v1.1.1/docs/velox/build_parameters/#gluten-jar-for-deployment"
- },"156": {
+ },"158": {
"doc": "v1.1.1/Build Parameters for Velox Backend",
"title": "v1.1.1/Build Parameters for Velox Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/build_parameters/",
"relUrl": "/archives/v1.1.1/docs/velox/build_parameters/"
- },"157": {
+ },"159": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Working with GCS",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/gcs/#working-with-gcs",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#working-with-gcs"
- },"158": {
+ },"160": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Installing the gcloud CLI",
"content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
"url": "/archives/v1.1.1/docs/velox/gcs/#installing-the-gcloud-cli",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#installing-the-gcloud-cli"
- },"159": {
+ },"161": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Configuring GCS using a user account",
"content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
"url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-user-account",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-user-account"
- },"160": {
+ },"162": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Configuring GCS using a credential file",
"content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
"url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-credential-file",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-credential-file"
- },"161": {
+ },"163": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Configuring GCS endpoints",
"content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
"url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-endpoints",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-endpoints"
- },"162": {
+ },"164": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "v1.1.1/Using GCS with Gluten",
"content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.1.1/docs/velox/gcs/",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/"
- },"163": {
+ },"165": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Working with S3",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/s3/#working-with-s3",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#working-with-s3"
- },"164": {
+ },"166": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Configuring S3 endpoint",
"content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
"url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-endpoint",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-endpoint"
- },"165": {
+ },"167": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Configuring S3 instance credentials",
"content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
"url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-instance-credentials",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-instance-credentials"
- },"166": {
+ },"168": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Configuring S3 IAM roles",
"content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
"url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-iam-roles",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-iam-roles"
- },"167": {
+ },"169": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Other authentatication methods are not supported yet",
"content": ". | AWS temporary credential | . ",
"url": "/archives/v1.1.1/docs/velox/s3/#other-authentatication-methods-are-not-supported-yet",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#other-authentatication-methods-are-not-supported-yet"
- },"168": {
+ },"170": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Log granularity of AWS C++ SDK in velox",
"content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
"url": "/archives/v1.1.1/docs/velox/s3/#log-granularity-of-aws-c-sdk-in-velox",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#log-granularity-of-aws-c-sdk-in-velox"
- },"169": {
+ },"171": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from HDFS/S3. The feature is very useful if remote storage is slow, e.g., reading from a public S3 bucket and stronger performance is desired. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage, and the future reading requests on already cached blocks will be serviced from local cache files. To enable the local caching feature, below configurations are required: . spark.g [...]
"url": "/archives/v1.1.1/docs/velox/s3/#local-caching-support",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#local-caching-support"
- },"170": {
+ },"172": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "v1.1.1/Using S3 with Gluten",
"content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.1.1/docs/velox/s3/",
"relUrl": "/archives/v1.1.1/docs/velox/s3/"
- },"171": {
+ },"173": {
"doc": "v1.1.1/Documentation",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.1.1/docs/#gluten-documents-by-version",
"relUrl": "/archives/v1.1.1/docs/#gluten-documents-by-version"
- },"172": {
+ },"174": {
"doc": "v1.1.1/Documentation",
"title": "v1.1.1/Documentation",
"content": " ",
"url": "/archives/v1.1.1/docs/",
"relUrl": "/archives/v1.1.1/docs/"
- },"173": {
+ },"175": {
"doc": "v1.1.1",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.1.1/#gluten-documents-by-version",
"relUrl": "/archives/v1.1.1/#gluten-documents-by-version"
- },"174": {
+ },"176": {
"doc": "v1.1.1",
"title": "v1.1.1",
"content": " ",
"url": "/archives/v1.1.1/",
"relUrl": "/archives/v1.1.1/"
- },"175": {
+ },"177": {
"doc": "Configuration(v1.2.1)",
"title": "Spark Configurations for Gluten Plugin",
"content": "There are many configurations could impact the Gluten Plugin performance and can be fine-tuned in Spark. You can add these configurations into spark-defaults.conf to enable or disable the setting. ",
"url": "/archives/v1.2.1/docs/configuration/#spark-configurations-for-gluten-plugin",
"relUrl": "/archives/v1.2.1/docs/configuration/#spark-configurations-for-gluten-plugin"
- },"176": {
+ },"178": {
"doc": "Configuration(v1.2.1)",
"title": "Spark parameters",
"content": "| Parameters | Description | Recommend Setting | . | spark.driver.extraClassPath | To add Gluten Plugin jar file in Spark Driver | /path/to/jar_file | . | spark.executor.extraClassPath | To add Gluten Plugin jar file in Spark Executor | /path/to/jar_file | . | spark.executor.memory | To set up how much memory to be used for Spark Executor. | | . | spark.memory.offHeap.size | To set up how much memory to be used for Java OffHeap. Please notice Gluten Plugin will leverage [...]
"url": "/archives/v1.2.1/docs/configuration/#spark-parameters",
"relUrl": "/archives/v1.2.1/docs/configuration/#spark-parameters"
- },"177": {
+ },"179": {
"doc": "Configuration(v1.2.1)",
"title": "Velox Parameters",
"content": "The following configurations are related to Velox settings. | Parameters | Description | Recommend Setting | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The m [...]
"url": "/archives/v1.2.1/docs/configuration/#velox-parameters",
"relUrl": "/archives/v1.2.1/docs/configuration/#velox-parameters"
- },"178": {
+ },"180": {
"doc": "Configuration(v1.2.1)",
"title": "Configuration(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/configuration/",
"relUrl": "/archives/v1.2.1/docs/configuration/"
- },"179": {
+ },"181": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Gluten CPP Core Guidelines",
"content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines"
- },"180": {
+ },"182": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Philosophy",
"content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#philosophy",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#philosophy"
- },"181": {
+ },"183": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Code Formatting",
"content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-formatting",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-formatting"
- },"182": {
+ },"184": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Naming Conventions",
"content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#naming-conventions",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#naming-conventions"
- },"183": {
+ },"185": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Designs",
"content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#designs",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#designs"
- },"184": {
+ },"186": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Source File & Header File",
"content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#source-file--header-file",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#source-file--header-file"
- },"185": {
+ },"187": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Class",
"content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#class",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#class"
- },"186": {
+ },"188": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Function",
"content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#function",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#function"
- },"187": {
+ },"189": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Variable",
"content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#variable",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#variable"
- },"188": {
+ },"190": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Constant",
"content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#constant",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#constant"
- },"189": {
+ },"191": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Macro",
"content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#macro",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#macro"
- },"190": {
+ },"192": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Namespace",
"content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#namespace",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#namespace"
- },"191": {
+ },"193": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Resource Management",
"content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#resource-management",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#resource-management"
- },"192": {
+ },"194": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Exception",
"content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#exception",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#exception"
- },"193": {
+ },"195": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Code Comment",
"content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-comment",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-comment"
- },"194": {
+ },"196": {
"doc": "CPP Code Style(v1.2.1)",
"title": "References",
"content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#references",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#references"
- },"195": {
+ },"197": {
"doc": "CPP Code Style(v1.2.1)",
"title": "CPP Code Style(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/"
- },"196": {
+ },"198": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to understand the key work of Gluten?",
"content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten"
- },"197": {
+ },"199": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to debug in Gluten?",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-debug-in-gluten",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-debug-in-gluten"
- },"198": {
+ },"200": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "1 How to debug C++",
"content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#1-how-to-debug-c",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#1-how-to-debug-c"
- },"199": {
+ },"201": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "2 How to debug plan validation process",
"content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process"
- },"200": {
+ },"202": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "3 How to debug Java/Scala",
"content": "wait to add . ",
"url": "/archives/v1.2.1/docs/developers/how-to-use/#3-how-to-debug-javascala",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#3-how-to-debug-javascala"
- },"201": {
+ },"203": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "4 How to debug with core-dump",
"content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
"url": "/archives/v1.2.1/docs/developers/how-to-use/#4-how-to-debug-with-core-dump",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#4-how-to-debug-with-core-dump"
- },"202": {
+ },"204": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to run TPC-H on Velox backend",
"content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox with docker by workflow, you can refer to velox_docker.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend"
- },"203": {
+ },"205": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to run TPC-DS",
"content": "wait to add . ",
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-ds",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-ds"
- },"204": {
+ },"206": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to track the memory exhaust problem",
"content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem"
- },"205": {
+ },"207": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How To Use Gluten(v1.2.1)",
"content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/"
- },"206": {
+ },"208": {
"doc": "How To Release(v1.2.1)",
"title": "How to Release",
"content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/archives/v1.2.1/docs/developers/how-to-release/#how-to-release",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-release/#how-to-release"
- },"207": {
+ },"209": {
"doc": "How To Release(v1.2.1)",
"title": "Release Process",
"content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
"url": "/archives/v1.2.1/docs/developers/how-to-release/#release-process",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-release/#release-process"
- },"208": {
+ },"210": {
"doc": "How To Release(v1.2.1)",
"title": "How To Release(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/how-to-release/",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-release/"
- },"209": {
+ },"211": {
"doc": "How To Scan Security issues(v1.2.1)",
"title": "How to scan the security issues",
"content": "This section outlines the steps to use tools to scan Apache Gluten (incubating) source code and make sure no vulnerability issues in the code. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/archives/v1.2.1/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues"
- },"210": {
+ },"212": {
"doc": "How To Scan Security issues(v1.2.1)",
"title": "Scan Security Process",
"content": "Before every Apache Gluten (incubating) release, we need to ensure there is no vulnerability issue in the source code. We use Trivy as the tool to scan all the security issues. | Install Trivy, please follow the steps to install Trivy: Trivy Installation . | Configuring Trivy, please follow the guide to configure Trivy for specific operation: Trivy Configuration . | Run Trivy File System Scan with the source code. Below is an example about how we run Trivy scan with Apach [...]
"url": "/archives/v1.2.1/docs/developers/how-to-security-scan/#scan-security-process",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/#scan-security-process"
- },"211": {
+ },"213": {
"doc": "How To Scan Security issues(v1.2.1)",
"title": "How To Scan Security issues(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/how-to-security-scan/",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/"
- },"212": {
+ },"214": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Generate Micro Benchmarks for Velox Backend",
"content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend"
- },"213": {
+ },"215": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Try the example",
"content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#try-the-example",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#try-the-example"
- },"214": {
+ },"216": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Generate Substrait plan and input for any query",
"content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query"
- },"215": {
+ },"217": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Save ouput to parquet to analyze",
"content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze"
- },"216": {
+ },"218": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Add shuffle write process",
"content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#add-shuffle-write-process",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#add-shuffle-write-process"
- },"217": {
+ },"219": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Simulate Spark with multiple processes and threads",
"content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads"
- },"218": {
+ },"220": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Micro Benchmarks for Velox Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/"
- },"219": {
+ },"221": {
"doc": "New To Gluten(v1.2.1)",
"title": "Environment",
"content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
"url": "/archives/v1.2.1/docs/developers/new-to/#environment",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#environment"
- },"220": {
+ },"222": {
"doc": "New To Gluten(v1.2.1)",
"title": "Openjdk8",
"content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#openjdk8",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#openjdk8"
- },"221": {
+ },"223": {
"doc": "New To Gluten(v1.2.1)",
"title": "Maven 3.6.3 or above",
"content": "Maven Dowload Page And then set the environment setting. ",
"url": "/archives/v1.2.1/docs/developers/new-to/#maven-363-or-above",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#maven-363-or-above"
- },"222": {
+ },"224": {
"doc": "New To Gluten(v1.2.1)",
"title": "GCC 9.4 or above",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/new-to/#gcc-94-or-above",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#gcc-94-or-above"
- },"223": {
+ },"225": {
"doc": "New To Gluten(v1.2.1)",
"title": "Compile gluten using debug mode",
"content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#compile-gluten-using-debug-mode",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#compile-gluten-using-debug-mode"
- },"224": {
+ },"226": {
"doc": "New To Gluten(v1.2.1)",
"title": "Java/scala code development with Intellij",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-development-with-intellij",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-development-with-intellij"
- },"225": {
+ },"227": {
"doc": "New To Gluten(v1.2.1)",
"title": "Linux intellij local debug",
"content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#linux-intellij-local-debug",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#linux-intellij-local-debug"
- },"226": {
+ },"228": {
"doc": "New To Gluten(v1.2.1)",
"title": "Windows/Mac intellij remote debug",
"content": "If you have Ultimate intellij, you can try to debug remotely. ",
"url": "/archives/v1.2.1/docs/developers/new-to/#windowsmac-intellij-remote-debug",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#windowsmac-intellij-remote-debug"
- },"227": {
+ },"229": {
"doc": "New To Gluten(v1.2.1)",
"title": "Set up gluten project",
"content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
"url": "/archives/v1.2.1/docs/developers/new-to/#set-up-gluten-project",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#set-up-gluten-project"
- },"228": {
+ },"230": {
"doc": "New To Gluten(v1.2.1)",
"title": "Java/Scala code style",
"content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-style",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-style"
- },"229": {
+ },"231": {
"doc": "New To Gluten(v1.2.1)",
"title": "CPP code development with Visual Studio Code",
"content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#cpp-code-development-with-visual-studio-code",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#cpp-code-development-with-visual-studio-code"
- },"230": {
+ },"232": {
"doc": "New To Gluten(v1.2.1)",
"title": "Usage",
"content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#usage",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#usage"
- },"231": {
+ },"233": {
"doc": "New To Gluten(v1.2.1)",
"title": "Usefule notes",
"content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#usefule-notes",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#usefule-notes"
- },"232": {
+ },"234": {
"doc": "New To Gluten(v1.2.1)",
"title": "Debug cpp code with coredump",
"content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-code-with-coredump",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-code-with-coredump"
- },"233": {
+ },"235": {
"doc": "New To Gluten(v1.2.1)",
"title": "Debug cpp with gdb",
"content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-with-gdb",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-with-gdb"
- },"234": {
+ },"236": {
"doc": "New To Gluten(v1.2.1)",
"title": "Run TPC-H and TPC-DS",
"content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_docker.yml . ",
"url": "/archives/v1.2.1/docs/developers/new-to/#run-tpc-h-and-tpc-ds",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#run-tpc-h-and-tpc-ds"
- },"235": {
+ },"237": {
"doc": "New To Gluten(v1.2.1)",
"title": "Run gluten+velox on clean machine",
"content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#run-glutenvelox-on-clean-machine",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#run-glutenvelox-on-clean-machine"
- },"236": {
+ },"238": {
"doc": "New To Gluten(v1.2.1)",
"title": "New To Gluten(v1.2.1)",
"content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
"url": "/archives/v1.2.1/docs/developers/new-to/",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/"
- },"237": {
+ },"239": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Substrait Modifications in Gluten",
"content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
"url": "/archives/v1.2.1/docs/developers/substrait/#substrait-modifications-in-gluten",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/#substrait-modifications-in-gluten"
- },"238": {
+ },"240": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Modifications to algebra.proto",
"content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
"url": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-algebraproto",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-algebraproto"
- },"239": {
+ },"241": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Modifications to type.proto",
"content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
"url": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-typeproto",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-typeproto"
- },"240": {
+ },"242": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Substrait Modifications(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/substrait/",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/"
- },"241": {
+ },"243": {
"doc": "Docker script for CentOS 7(v1.2.1)",
"title": "Docker script for CentOS 7(v1.2.1)",
"content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git236 \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ autoconf \\ autoconf-archive \\ automake \\ perl-IPC-Cmd \\ libicu-devel \\ [...]
"url": "/archives/v1.2.1/docs/developers/docker-centos7/",
"relUrl": "/archives/v1.2.1/docs/developers/docker-centos7/"
- },"242": {
+ },"244": {
"doc": "Docker script for CentOS 8(v1.2.1)",
"title": "Docker script for CentOS 8(v1.2.1)",
"content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
"url": "/archives/v1.2.1/docs/developers/docker-centos8/",
"relUrl": "/archives/v1.2.1/docs/developers/docker-centos8/"
- },"243": {
+ },"245": {
"doc": "Docker script for Ubuntu 22.04/20.04(v1.2.1)",
"title": "Docker script for Ubuntu 22.04/20.04(v1.2.1)",
"content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
"url": "/archives/v1.2.1/docs/developers/docker-ubuntu/",
"relUrl": "/archives/v1.2.1/docs/developers/docker-ubuntu/"
- },"244": {
+ },"246": {
"doc": "Developers(v1.2.1)",
"title": "Gluten Developers",
"content": "This document provides a developer overview of the project and covers the following topics: . ",
"url": "/archives/v1.2.1/docs/developers/#gluten-developers",
"relUrl": "/archives/v1.2.1/docs/developers/#gluten-developers"
- },"245": {
+ },"247": {
"doc": "Developers(v1.2.1)",
"title": "Developers(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/",
"relUrl": "/archives/v1.2.1/docs/developers/"
- },"246": {
+ },"248": {
"doc": "Velox Function Development(v1.2.1)",
"title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
"content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
"url": "/archives/v1.2.1/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
"relUrl": "/archives/v1.2.1/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"247": {
+ },"249": {
"doc": "Velox Function Development(v1.2.1)",
"title": "Velox Function Development(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/velox-function-dev/",
"relUrl": "/archives/v1.2.1/docs/developers/velox-function-dev/"
- },"248": {
+ },"250": {
"doc": "Getting start with ClickHouse Backend(v1.2.1)",
"title": "ClickHouse Backend",
"content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
"url": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/#clickhouse-backend",
"relUrl": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/#clickhouse-backend"
- },"249": {
+ },"251": {
"doc": "Getting start with ClickHouse Backend(v1.2.1)",
"title": "Getting start with ClickHouse Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/",
"relUrl": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/"
- },"250": {
+ },"252": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Supported Version",
"content": "| Type | Version | . | Spark | 3.2.2, 3.3.1, 3.4.2, 3.5.1 | . | OS | Ubuntu20.04/22.04, Centos7/8 | . | jdk | openjdk8/jdk17 | . | scala | 2.12 | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#supported-version",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#supported-version"
- },"251": {
+ },"253": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Prerequisite",
"content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos7/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.2 and 3.5.1. We need to set up the JAVA_HOME env. Currently, Gluten supports java 8 and java 17. For x86_64 . ## make sure jdk8 is used export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#prerequisite",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#prerequisite"
- },"252": {
+ },"254": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with Velox Backend",
"content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . First time build for all supported spark versions./dev/buildbundle-veloxbe.sh . After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and setting up build dependencies./dev/buildbundle-veloxbe.sh --enable_ep [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-velox-backend",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-velox-backend"
- },"253": {
+ },"255": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Dependency library deployment",
"content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#dependency-library-deployment",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#dependency-library-deployment"
- },"254": {
+ },"256": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "HDFS support",
"content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#hdfs-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#hdfs-support"
- },"255": {
+ },"257": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Azure Blob File System (ABFS) support",
"content": "Velox supports ABFS with the open source Azure SDK for C++ and Gluten uses the Velox ABFS connector to connect with ABFS. The build option for ABFS (enable_abfs) must be set to enable this feature as listed below. cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_abfs=ON . Please refer Velox ABFS part for more detailed configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#azure-blob-file-system-abfs-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#azure-blob-file-system-abfs-support"
- },"256": {
+ },"258": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "AWS S3 support",
"content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#aws-s3-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#aws-s3-support"
- },"257": {
+ },"259": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Celeborn support",
"content": "Gluten with velox backend supports Celeborn as remote shuffle service. Currently, the supported Celeborn versions are 0.3.x, 0.4.x and 0.5.0. Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable celeborn profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests . Then add the Gluten and Spa [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#celeborn-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#celeborn-support"
- },"258": {
+ },"260": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Uniffle support",
"content": "Uniffle with velox backend supports Uniffle as remote shuffle service. Currently, the supported Uniffle versions are 0.8.0. First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with uniffle. When compiling the Gluten Java module, it’s required to enable uniffle profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests . Then add the Uniffle and Spark Celeborn Client packages to your Spark application’s classpath(usu [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#uniffle-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#uniffle-support"
- },"259": {
+ },"261": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "DeltaLake Support",
"content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta-XX-SNAPSHOT.jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with vel [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#deltalake-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#deltalake-support"
- },"260": {
+ },"262": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Iceberg Support",
"content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan’s fallback. After the two steps, you can query [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#iceberg-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#iceberg-support"
- },"261": {
+ },"263": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Coverage",
"content": "Spark3.3 has 387 functions in total. ~240 are commonly used. To get the support status of all Spark built-in functions, please refer to Velox Backend’s Supported Operators & Functions. Velox doesn’t support ANSI mode), so as Gluten. Once ANSI mode is enabled in Spark config, Gluten will fallback to Vanilla Spark. To identify what can be offloaded in a query and detailed fallback reasons, user can follow below steps to retrieve corresponding logs. 1) Enable Gluten by p [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#coverage",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#coverage"
- },"262": {
+ },"264": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Spill (Experimental)",
"content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#spill-experimental",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#spill-experimental"
- },"263": {
+ },"265": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": "Please check the VeloxNativeUDF.md for more detailed usage and configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"264": {
+ },"266": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "High-Bandwidth Memory (HBM) support",
"content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#high-bandwidth-memory-hbm-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#high-bandwidth-memory-hbm-support"
- },"265": {
+ },"267": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with HBM",
"content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.2 and 3.5.1./dev/buildbundle-veloxbe.sh -- [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-hbm",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-hbm"
- },"266": {
+ },"268": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Configure and enable HBM in Spark Application",
"content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application"
- },"267": {
+ },"269": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Intel® QuickAssist Technology (QAT) support",
"content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-quickassist-technology-qat-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-quickassist-technology-qat-support"
- },"268": {
+ },"270": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Software Requirements",
"content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#software-requirements",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#software-requirements"
- },"269": {
+ },"271": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with QAT",
"content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-qat",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-qat"
- },"270": {
+ },"272": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Enable QAT with Gzip/Zstd for shuffle compression",
"content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression"
- },"271": {
+ },"273": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "QAT driver references",
"content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#qat-driver-references",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#qat-driver-references"
- },"272": {
+ },"274": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
"content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support"
- },"273": {
+ },"275": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with IAA",
"content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-iaa",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-iaa"
- },"274": {
+ },"276": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Enable IAA with Gzip Compression for shuffle compression",
"content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression"
- },"275": {
+ },"277": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "IAA references",
"content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#iaa-references",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#iaa-references"
- },"276": {
+ },"278": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
"content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
- },"277": {
+ },"279": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Data preparation",
"content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#data-preparation",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#data-preparation"
- },"278": {
+ },"280": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Submit the Spark SQL job",
"content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#submit-the-spark-sql-job",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#submit-the-spark-sql-job"
- },"279": {
+ },"281": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Result",
"content": "wholestagetransformer indicates that the offloading works. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#result",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#result"
- },"280": {
+ },"282": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Performance",
"content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#performance",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#performance"
- },"281": {
+ },"283": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "External reference setup",
"content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#external-reference-setup",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#external-reference-setup"
- },"282": {
+ },"284": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Gluten UI",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-ui",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-ui"
- },"283": {
+ },"285": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Gluten event",
"content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-event",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-event"
- },"284": {
+ },"286": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "SQL tab",
"content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#sql-tab",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#sql-tab"
- },"285": {
+ },"287": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "History server",
"content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#history-server",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#history-server"
- },"286": {
+ },"288": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Native plan string",
"content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-string",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-string"
- },"287": {
+ },"289": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Native plan with stats",
"content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-with-stats",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-with-stats"
- },"288": {
+ },"290": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Gluten Implicits",
"content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-implicits",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-implicits"
- },"289": {
+ },"291": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Getting Start with Velox Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/"
- },"290": {
+ },"292": {
"doc": "Using ABFS with Gluten(v1.2.1)",
"title": "Working with ABFS",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/abfs/#working-with-abfs",
"relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#working-with-abfs"
- },"291": {
+ },"293": {
"doc": "Using ABFS with Gluten(v1.2.1)",
"title": "Configuring ABFS Access Token",
"content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
"url": "/archives/v1.2.1/docs/getting-started/abfs/#configuring-abfs-access-token",
"relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#configuring-abfs-access-token"
- },"292": {
+ },"294": {
"doc": "Using ABFS with Gluten(v1.2.1)",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/abfs/#local-caching-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#local-caching-support"
- },"293": {
+ },"295": {
"doc": "Using ABFS with Gluten(v1.2.1)",
"title": "Using ABFS with Gluten(v1.2.1)",
"content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
"url": "/archives/v1.2.1/docs/getting-started/abfs/",
"relUrl": "/archives/v1.2.1/docs/getting-started/abfs/"
- },"294": {
+ },"296": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Working with GCS",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#working-with-gcs",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#working-with-gcs"
- },"295": {
+ },"297": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Installing the gcloud CLI",
"content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#installing-the-gcloud-cli",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#installing-the-gcloud-cli"
- },"296": {
+ },"298": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS using a user account",
"content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-user-account",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-user-account"
- },"297": {
+ },"299": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS using a credential file",
"content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file"
- },"298": {
+ },"300": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS endpoints",
"content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-endpoints",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-endpoints"
- },"299": {
+ },"301": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS max retry count",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-count",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-count"
- },"300": {
+ },"302": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS max retry time",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retring (10s, 1m, etc). ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-time",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-time"
- },"301": {
+ },"303": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Using GCS with Gluten(v1.2.1)",
"content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/"
- },"302": {
+ },"304": {
"doc": "Velox Local Caching(v1.2.1)",
"title": "Velox Local Caching(v1.2.1)",
"content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
"url": "/archives/v1.2.1/docs/getting-started/localcache/",
"relUrl": "/archives/v1.2.1/docs/getting-started/localcache/"
- },"303": {
+ },"305": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Working with S3",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#working-with-s3",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#working-with-s3"
- },"304": {
+ },"306": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Configuring S3 endpoint",
"content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
"url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-endpoint",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-endpoint"
- },"305": {
+ },"307": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Configuring S3 instance credentials",
"content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-instance-credentials",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-instance-credentials"
- },"306": {
+ },"308": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Configuring S3 IAM roles",
"content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-iam-roles",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-iam-roles"
- },"307": {
+ },"309": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Other authentatication methods are not supported yet",
"content": ". | AWS temporary credential | . ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet"
- },"308": {
+ },"310": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Log granularity of AWS C++ SDK in velox",
"content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox"
- },"309": {
+ },"311": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#local-caching-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#local-caching-support"
- },"310": {
+ },"312": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Using S3 with Gluten(v1.2.1)",
"content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.2.1/docs/getting-started/s3/",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/"
- },"311": {
+ },"313": {
"doc": "Build Parameters for Velox Backend(v1.2.1)",
"title": "Build Parameters",
"content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | OFF | . | build_proto [...]
"url": "/archives/v1.2.1/docs/getting-started/build-guide/#build-parameters",
"relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/#build-parameters"
- },"312": {
+ },"314": {
"doc": "Build Parameters for Velox Backend(v1.2.1)",
"title": "Gluten Jar for Deployment",
"content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . | 3.5.1 | 3.5 | 2.12 | . ",
"url": "/archives/v1.2.1/docs/getting-started/build-guide/#gluten-jar-for-deployment",
"relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/#gluten-jar-for-deployment"
- },"313": {
+ },"315": {
"doc": "Build Parameters for Velox Backend(v1.2.1)",
"title": "Build Parameters for Velox Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/build-guide/",
"relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/"
- },"314": {
+ },"316": {
"doc": "Getting-Started(v1.2.1)",
"title": "Getting Started with Gluten for Apache Spark",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/#getting-started-with-gluten-for-apache-spark",
"relUrl": "/archives/v1.2.1/docs/getting-started/#getting-started-with-gluten-for-apache-spark"
- },"315": {
+ },"317": {
"doc": "Getting-Started(v1.2.1)",
"title": "Getting-Started(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/",
"relUrl": "/archives/v1.2.1/docs/getting-started/"
- },"316": {
+ },"318": {
"doc": "Documentations(v1.2.1)",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.2.1/docs/#gluten-documents-by-version",
"relUrl": "/archives/v1.2.1/docs/#gluten-documents-by-version"
- },"317": {
+ },"319": {
"doc": "Documentations(v1.2.1)",
"title": "Documentations(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/",
"relUrl": "/archives/v1.2.1/docs/"
- },"318": {
+ },"320": {
"doc": "Velox Backend(v1.2.1)",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/#gluten-documents-by-version",
"relUrl": "/archives/v1.2.1/docs/velox-backend/#gluten-documents-by-version"
- },"319": {
+ },"321": {
"doc": "Velox Backend(v1.2.1)",
"title": "Velox Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/"
- },"320": {
+ },"322": {
"doc": "Limitations(v1.2.1)",
"title": "Velox Backend Limitations",
"content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
"url": "/archives/v1.2.1/docs/velox-backend/limitations/#velox-backend-limitations",
"relUrl": "/archives/v1.2.1/docs/velox-backend/limitations/#velox-backend-limitations"
- },"321": {
+ },"323": {
"doc": "Limitations(v1.2.1)",
"title": "Limitations(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/limitations/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/limitations/"
- },"322": {
+ },"324": {
"doc": "Supported Operators & Functions(v1.2.1)",
"title": "The Operators and Functions Support Progress",
"content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. The total number of fun [...]
"url": "/archives/v1.2.1/docs/velox-backend/support/#the-operators-and-functions-support-progress",
"relUrl": "/archives/v1.2.1/docs/velox-backend/support/#the-operators-and-functions-support-progress"
- },"323": {
+ },"325": {
"doc": "Supported Operators & Functions(v1.2.1)",
"title": "Supported Operators & Functions(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/support/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/support/"
- },"324": {
+ },"326": {
"doc": "Troubleshooting(v1.2.1)",
"title": "Velox Backend Troubleshooting",
"content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
"url": "/archives/v1.2.1/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting",
"relUrl": "/archives/v1.2.1/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting"
- },"325": {
+ },"327": {
"doc": "Troubleshooting(v1.2.1)",
"title": "Troubleshooting(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/troubleshooting/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/troubleshooting/"
- },"326": {
+ },"328": {
"doc": "Velox UDF(v1.2.1)",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"327": {
+ },"329": {
"doc": "Velox UDF(v1.2.1)",
"title": "Introduction",
"content": "Velox backend supports User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#introduction",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#introduction"
- },"328": {
+ },"330": {
"doc": "Velox UDF(v1.2.1)",
"title": "Create and Build UDF/UDAF library",
"content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
"url": "/archives/v1.2.1/docs/velox-backend/udf/#create-and-build-udfudaf-library",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#create-and-build-udfudaf-library"
- },"329": {
+ },"331": {
"doc": "Velox UDF(v1.2.1)",
"title": "Using UDF/UDAF in Gluten",
"content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the library paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
"url": "/archives/v1.2.1/docs/velox-backend/udf/#using-udfudaf-in-gluten",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#using-udfudaf-in-gluten"
- },"330": {
+ },"332": {
"doc": "Velox UDF(v1.2.1)",
"title": "Try the example",
"content": "We provided Velox UDF examples in file MyUDF.cc and UDAF examples in file MyUDAF.cc. You need to build the gluten project with --build_example=ON to get the example libraries./dev/buildbundle-veloxbe.sh --build_examples=ON . Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ . Start spark-shell or spark-sql with below configuration . # Use the `--files` option to upload a library and configure its relative path --files /path/to/glute [...]
"url": "/archives/v1.2.1/docs/velox-backend/udf/#try-the-example",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#try-the-example"
- },"331": {
+ },"333": {
"doc": "Velox UDF(v1.2.1)",
"title": "Configurations",
"content": "| Parameters | Description | . | spark.gluten.sql.columnar.backend.velox.udfLibraryPaths | Path to the udf/udaf libraries. | . | spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths | Path to the udf/udaf libraries on driver node. Only applicable on yarn-client mode. | . | spark.gluten.sql.columnar.backend.velox.udfAllowTypeConversion | Whether to inject possible cast to convert mismatched data types from input to one registered signatures. | . ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#configurations",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#configurations"
- },"332": {
+ },"334": {
"doc": "Velox UDF(v1.2.1)",
"title": "Pandas UDFs (a.k.a. Vectorized UDFs)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs"
- },"333": {
+ },"335": {
"doc": "Velox UDF(v1.2.1)",
"title": "Introduction",
"content": "Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer doc. ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#introduction-1",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#introduction-1"
- },"334": {
+ },"336": {
"doc": "Velox UDF(v1.2.1)",
"title": "Using Pandas UDFs in Gluten with Velox Backend",
"content": "Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: . pip3 install pyspark==$SPARK_VERSION cython pip3 install pandas pyarrow . Gluten provides a config to control enable ColumnarArrowEvalPython or not, with true as defalt. spark.gluten.sql.columnar.arrowUdf . Then take following PySpark code for example: . from pyspark.sql.functions import pandas_udf, PandasUDFType import pyspark.sql.functions as [...]
"url": "/archives/v1.2.1/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend"
- },"335": {
+ },"337": {
"doc": "Velox UDF(v1.2.1)",
"title": "Velox UDF(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/"
- },"336": {
+ },"338": {
"doc": "v1.2.1",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.2.1/#gluten-documents-by-version",
"relUrl": "/archives/v1.2.1/#gluten-documents-by-version"
- },"337": {
+ },"339": {
"doc": "v1.2.1",
"title": "v1.2.1",
"content": " ",
"url": "/archives/v1.2.1/",
"relUrl": "/archives/v1.2.1/"
- },"338": {
+ },"340": {
"doc": "Configuration",
"title": "Spark Configurations for Gluten Plugin",
"content": "There are many configurations could impact the Gluten Plugin performance and can be fine-tuned in Spark. You can add these configurations into spark-defaults.conf to enable or disable the setting. ",
"url": "/archives/v1.3.0/configuration#spark-configurations-for-gluten-plugin",
"relUrl": "/archives/v1.3.0/configuration#spark-configurations-for-gluten-plugin"
- },"339": {
+ },"341": {
"doc": "Configuration",
"title": "Spark parameters",
"content": "| Parameters | Description | Recommend Setting | . | spark.driver.extraClassPath | To add Gluten Plugin jar file in Spark Driver | /path/to/jar_file | . | spark.executor.extraClassPath | To add Gluten Plugin jar file in Spark Executor | /path/to/jar_file | . | spark.executor.memory | To set up how much memory to be used for Spark Executor. | | . | spark.memory.offHeap.size | To set up how much memory to be used for Java OffHeap. Please notice Gluten Plugin will leverage [...]
"url": "/archives/v1.3.0/configuration#spark-parameters",
"relUrl": "/archives/v1.3.0/configuration#spark-parameters"
- },"340": {
+ },"342": {
"doc": "Configuration",
"title": "Velox Parameters",
"content": "The following configurations are related to Velox settings. | Parameters | Description | Recommend Setting | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The m [...]
"url": "/archives/v1.3.0/configuration#velox-parameters",
"relUrl": "/archives/v1.3.0/configuration#velox-parameters"
- },"341": {
+ },"343": {
"doc": "Configuration",
"title": "Configuration",
"content": " ",
"url": "/archives/v1.3.0/configuration",
"relUrl": "/archives/v1.3.0/configuration"
- },"342": {
+ },"344": {
"doc": "CPP Code Style",
"title": "Gluten CPP Core Guidelines",
"content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#gluten-cpp-core-guidelines",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#gluten-cpp-core-guidelines"
- },"343": {
+ },"345": {
"doc": "CPP Code Style",
"title": "Philosophy",
"content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#philosophy",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#philosophy"
- },"344": {
+ },"346": {
"doc": "CPP Code Style",
"title": "Code Formatting",
"content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#code-formatting",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#code-formatting"
- },"345": {
+ },"347": {
"doc": "CPP Code Style",
"title": "Naming Conventions",
"content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#naming-conventions",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#naming-conventions"
- },"346": {
+ },"348": {
"doc": "CPP Code Style",
"title": "Designs",
"content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#designs",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#designs"
- },"347": {
+ },"349": {
"doc": "CPP Code Style",
"title": "Source File & Header File",
"content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#source-file--header-file",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#source-file--header-file"
- },"348": {
+ },"350": {
"doc": "CPP Code Style",
"title": "Class",
"content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#class",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#class"
- },"349": {
+ },"351": {
"doc": "CPP Code Style",
"title": "Function",
"content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#function",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#function"
- },"350": {
+ },"352": {
"doc": "CPP Code Style",
"title": "Variable",
"content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#variable",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#variable"
- },"351": {
+ },"353": {
"doc": "CPP Code Style",
"title": "Constant",
"content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#constant",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#constant"
- },"352": {
+ },"354": {
"doc": "CPP Code Style",
"title": "Macro",
"content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#macro",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#macro"
- },"353": {
+ },"355": {
"doc": "CPP Code Style",
"title": "Namespace",
"content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#namespace",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#namespace"
- },"354": {
+ },"356": {
"doc": "CPP Code Style",
"title": "Resource Management",
"content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#resource-management",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#resource-management"
- },"355": {
+ },"357": {
"doc": "CPP Code Style",
"title": "Exception",
"content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#exception",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#exception"
- },"356": {
+ },"358": {
"doc": "CPP Code Style",
"title": "Code Comment",
"content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#code-comment",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#code-comment"
- },"357": {
+ },"359": {
"doc": "CPP Code Style",
"title": "References",
"content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#references",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#references"
- },"358": {
+ },"360": {
"doc": "CPP Code Style",
"title": "CPP Code Style",
"content": " ",
"url": "/archives/v1.3.0/developers/cpp-code-style/",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/"
- },"359": {
+ },"361": {
"doc": "How To Use Gluten",
"title": "How to understand the key work of Gluten?",
"content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-understand-the-key-work-of-gluten",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-understand-the-key-work-of-gluten"
- },"360": {
+ },"362": {
"doc": "How To Use Gluten",
"title": "How to debug in Gluten?",
"content": " ",
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-debug-in-gluten",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-debug-in-gluten"
- },"361": {
+ },"363": {
"doc": "How To Use Gluten",
"title": "1 How to debug C++",
"content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
"url": "/archives/v1.3.0/developers/how-to-use/#1-how-to-debug-c",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#1-how-to-debug-c"
- },"362": {
+ },"364": {
"doc": "How To Use Gluten",
"title": "2 How to debug plan validation process",
"content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
"url": "/archives/v1.3.0/developers/how-to-use/#2-how-to-debug-plan-validation-process",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#2-how-to-debug-plan-validation-process"
- },"363": {
+ },"365": {
"doc": "How To Use Gluten",
"title": "3 How to debug Java/Scala",
"content": "wait to add . ",
"url": "/archives/v1.3.0/developers/how-to-use/#3-how-to-debug-javascala",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#3-how-to-debug-javascala"
- },"364": {
+ },"366": {
"doc": "How To Use Gluten",
"title": "4 How to debug with core-dump",
"content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
"url": "/archives/v1.3.0/developers/how-to-use/#4-how-to-debug-with-core-dump",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#4-how-to-debug-with-core-dump"
- },"365": {
+ },"367": {
"doc": "How To Use Gluten",
"title": "How to run TPC-H on Velox backend",
"content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox with docker by workflow, you can refer to velox_docker.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, [...]
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend"
- },"366": {
+ },"368": {
"doc": "How To Use Gluten",
"title": "How to run TPC-DS",
"content": "wait to add . ",
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-ds",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-ds"
- },"367": {
+ },"369": {
"doc": "How To Use Gluten",
"title": "How to track the memory exhaust problem",
"content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-track-the-memory-exhaust-problem",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-track-the-memory-exhaust-problem"
- },"368": {
+ },"370": {
"doc": "How To Use Gluten",
"title": "How To Use Gluten",
"content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
"url": "/archives/v1.3.0/developers/how-to-use/",
"relUrl": "/archives/v1.3.0/developers/how-to-use/"
- },"369": {
+ },"371": {
"doc": "How To Release",
"title": "How to Release",
"content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/docs/developers/how-to-release/#how-to-release",
"relUrl": "/docs/developers/how-to-release/#how-to-release"
- },"370": {
+ },"372": {
"doc": "How To Release",
"title": "Release Process",
"content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
"url": "/docs/developers/how-to-release/#release-process",
"relUrl": "/docs/developers/how-to-release/#release-process"
- },"371": {
+ },"373": {
"doc": "How To Release",
"title": "How To Release",
"content": " ",
"url": "/docs/developers/how-to-release/",
"relUrl": "/docs/developers/how-to-release/"
- },"372": {
+ },"374": {
"doc": "How To Scan Security issues",
"title": "How to scan the security issues",
"content": "This section outlines the steps to use tools to scan Apache Gluten (incubating) source code and make sure no vulnerability issues in the code. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/archives/v1.3.0/developers/how-to-security-scan/#how-to-scan-the-security-issues",
"relUrl": "/archives/v1.3.0/developers/how-to-security-scan/#how-to-scan-the-security-issues"
- },"373": {
+ },"375": {
"doc": "How To Scan Security issues",
"title": "Scan Security Process",
"content": "Before every Apache Gluten (incubating) release, we need to ensure there is no vulnerability issue in the source code. We use Trivy as the tool to scan all the security issues. | Install Trivy, please follow the steps to install Trivy: Trivy Installation . | Configuring Trivy, please follow the guide to configure Trivy for specific operation: Trivy Configuration . | Run Trivy File System Scan with the source code. Below is an example about how we run Trivy scan with Apach [...]
"url": "/archives/v1.3.0/developers/how-to-security-scan/#scan-security-process",
"relUrl": "/archives/v1.3.0/developers/how-to-security-scan/#scan-security-process"
- },"374": {
+ },"376": {
"doc": "How To Scan Security issues",
"title": "How To Scan Security issues",
"content": " ",
"url": "/archives/v1.3.0/developers/how-to-security-scan/",
"relUrl": "/archives/v1.3.0/developers/how-to-security-scan/"
- },"375": {
+ },"377": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Generate Micro Benchmarks for Velox Backend",
"content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend"
- },"376": {
+ },"378": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Try the example",
"content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#try-the-example",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#try-the-example"
- },"377": {
+ },"379": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Generate Substrait plan and input for any query",
"content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query"
- },"378": {
+ },"380": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Save ouput to parquet to analyze",
"content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
"url": "/archives/v1.3.0/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze"
- },"379": {
+ },"381": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Add shuffle write process",
"content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#add-shuffle-write-process",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#add-shuffle-write-process"
- },"380": {
+ },"382": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Simulate Spark with multiple processes and threads",
"content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads"
- },"381": {
+ },"383": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Micro Benchmarks for Velox Backend",
"content": " ",
"url": "/archives/v1.3.0/developers/microbenchmarks/",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/"
- },"382": {
+ },"384": {
"doc": "New To Gluten",
"title": "Environment",
"content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
"url": "/archives/v1.3.0/developers/new-to/#environment",
"relUrl": "/archives/v1.3.0/developers/new-to/#environment"
- },"383": {
+ },"385": {
"doc": "New To Gluten",
"title": "Openjdk8",
"content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
"url": "/archives/v1.3.0/developers/new-to/#openjdk8",
"relUrl": "/archives/v1.3.0/developers/new-to/#openjdk8"
- },"384": {
+ },"386": {
"doc": "New To Gluten",
"title": "Maven 3.6.3 or above",
"content": "Maven Dowload Page And then set the environment setting. ",
"url": "/archives/v1.3.0/developers/new-to/#maven-363-or-above",
"relUrl": "/archives/v1.3.0/developers/new-to/#maven-363-or-above"
- },"385": {
+ },"387": {
"doc": "New To Gluten",
"title": "GCC 9.4 or above",
"content": " ",
"url": "/archives/v1.3.0/developers/new-to/#gcc-94-or-above",
"relUrl": "/archives/v1.3.0/developers/new-to/#gcc-94-or-above"
- },"386": {
+ },"388": {
"doc": "New To Gluten",
"title": "Compile gluten using debug mode",
"content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
"url": "/archives/v1.3.0/developers/new-to/#compile-gluten-using-debug-mode",
"relUrl": "/archives/v1.3.0/developers/new-to/#compile-gluten-using-debug-mode"
- },"387": {
+ },"389": {
"doc": "New To Gluten",
"title": "Java/scala code development with Intellij",
"content": " ",
"url": "/archives/v1.3.0/developers/new-to/#javascala-code-development-with-intellij",
"relUrl": "/archives/v1.3.0/developers/new-to/#javascala-code-development-with-intellij"
- },"388": {
+ },"390": {
"doc": "New To Gluten",
"title": "Linux intellij local debug",
"content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
"url": "/archives/v1.3.0/developers/new-to/#linux-intellij-local-debug",
"relUrl": "/archives/v1.3.0/developers/new-to/#linux-intellij-local-debug"
- },"389": {
+ },"391": {
"doc": "New To Gluten",
"title": "Windows/Mac intellij remote debug",
"content": "If you have Ultimate intellij, you can try to debug remotely. ",
"url": "/archives/v1.3.0/developers/new-to/#windowsmac-intellij-remote-debug",
"relUrl": "/archives/v1.3.0/developers/new-to/#windowsmac-intellij-remote-debug"
- },"390": {
+ },"392": {
"doc": "New To Gluten",
"title": "Set up gluten project",
"content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
"url": "/archives/v1.3.0/developers/new-to/#set-up-gluten-project",
"relUrl": "/archives/v1.3.0/developers/new-to/#set-up-gluten-project"
- },"391": {
+ },"393": {
"doc": "New To Gluten",
"title": "Java/Scala code style",
"content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
"url": "/archives/v1.3.0/developers/new-to/#javascala-code-style",
"relUrl": "/archives/v1.3.0/developers/new-to/#javascala-code-style"
- },"392": {
+ },"394": {
"doc": "New To Gluten",
"title": "CPP code development with Visual Studio Code",
"content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
"url": "/archives/v1.3.0/developers/new-to/#cpp-code-development-with-visual-studio-code",
"relUrl": "/archives/v1.3.0/developers/new-to/#cpp-code-development-with-visual-studio-code"
- },"393": {
+ },"395": {
"doc": "New To Gluten",
"title": "Usage",
"content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
"url": "/archives/v1.3.0/developers/new-to/#usage",
"relUrl": "/archives/v1.3.0/developers/new-to/#usage"
- },"394": {
+ },"396": {
"doc": "New To Gluten",
"title": "Usefule notes",
"content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
"url": "/archives/v1.3.0/developers/new-to/#usefule-notes",
"relUrl": "/archives/v1.3.0/developers/new-to/#usefule-notes"
- },"395": {
+ },"397": {
"doc": "New To Gluten",
"title": "Debug cpp code with coredump",
"content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
"url": "/archives/v1.3.0/developers/new-to/#debug-cpp-code-with-coredump",
"relUrl": "/archives/v1.3.0/developers/new-to/#debug-cpp-code-with-coredump"
- },"396": {
+ },"398": {
"doc": "New To Gluten",
"title": "Debug cpp with gdb",
"content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
"url": "/archives/v1.3.0/developers/new-to/#debug-cpp-with-gdb",
"relUrl": "/archives/v1.3.0/developers/new-to/#debug-cpp-with-gdb"
- },"397": {
+ },"399": {
"doc": "New To Gluten",
"title": "Run TPC-H and TPC-DS",
"content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_docker.yml . ",
"url": "/archives/v1.3.0/developers/new-to/#run-tpc-h-and-tpc-ds",
"relUrl": "/archives/v1.3.0/developers/new-to/#run-tpc-h-and-tpc-ds"
- },"398": {
+ },"400": {
"doc": "New To Gluten",
"title": "Run gluten+velox on clean machine",
"content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
"url": "/archives/v1.3.0/developers/new-to/#run-glutenvelox-on-clean-machine",
"relUrl": "/archives/v1.3.0/developers/new-to/#run-glutenvelox-on-clean-machine"
- },"399": {
+ },"401": {
"doc": "New To Gluten",
"title": "New To Gluten",
"content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
"url": "/archives/v1.3.0/developers/new-to/",
"relUrl": "/archives/v1.3.0/developers/new-to/"
- },"400": {
+ },"402": {
"doc": "Substrait Modifications",
"title": "Substrait Modifications in Gluten",
"content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
"url": "/archives/v1.3.0/developers/substrait/#substrait-modifications-in-gluten",
"relUrl": "/archives/v1.3.0/developers/substrait/#substrait-modifications-in-gluten"
- },"401": {
+ },"403": {
"doc": "Substrait Modifications",
"title": "Modifications to algebra.proto",
"content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
"url": "/archives/v1.3.0/developers/substrait/#modifications-to-algebraproto",
"relUrl": "/archives/v1.3.0/developers/substrait/#modifications-to-algebraproto"
- },"402": {
+ },"404": {
"doc": "Substrait Modifications",
"title": "Modifications to type.proto",
"content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
"url": "/archives/v1.3.0/developers/substrait/#modifications-to-typeproto",
"relUrl": "/archives/v1.3.0/developers/substrait/#modifications-to-typeproto"
- },"403": {
+ },"405": {
"doc": "Substrait Modifications",
"title": "Substrait Modifications",
"content": " ",
"url": "/archives/v1.3.0/developers/substrait/",
"relUrl": "/archives/v1.3.0/developers/substrait/"
- },"404": {
+ },"406": {
"doc": "Docker script for CentOS 7",
"title": "Docker script for CentOS 7",
"content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git236 \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ autoconf \\ autoconf-archive \\ automake \\ perl-IPC-Cmd \\ libicu-devel \\ [...]
"url": "/archives/v1.3.0/developers/docker-centos7/",
"relUrl": "/archives/v1.3.0/developers/docker-centos7/"
- },"405": {
+ },"407": {
"doc": "Docker script for CentOS 8",
"title": "Docker script for CentOS 8",
"content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
"url": "/archives/v1.3.0/developers/docker-centos8/",
"relUrl": "/archives/v1.3.0/developers/docker-centos8/"
- },"406": {
+ },"408": {
"doc": "Docker script for Ubuntu 22.04/20.04",
"title": "Docker script for Ubuntu 22.04/20.04",
"content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
"url": "/archives/v1.3.0/developers/docker-ubuntu/",
"relUrl": "/archives/v1.3.0/developers/docker-ubuntu/"
- },"407": {
+ },"409": {
"doc": "Developers",
"title": "Gluten Developers",
"content": "This document provides a developer overview of the project and covers the following topics: . ",
"url": "/archives/v1.3.0/developers#gluten-developers",
"relUrl": "/archives/v1.3.0/developers#gluten-developers"
- },"408": {
+ },"410": {
"doc": "Developers",
"title": "Developers",
"content": " ",
"url": "/archives/v1.3.0/developers",
"relUrl": "/archives/v1.3.0/developers"
- },"409": {
+ },"411": {
"doc": "Velox Function Development",
"title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
"content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
"url": "/archives/v1.3.0/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
"relUrl": "/archives/v1.3.0/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"410": {
+ },"412": {
"doc": "Velox Function Development",
"title": "Velox Function Development",
"content": " ",
"url": "/archives/v1.3.0/developers/velox-function-dev/",
"relUrl": "/archives/v1.3.0/developers/velox-function-dev/"
- },"411": {
+ },"413": {
"doc": "Getting start with ClickHouse Backend",
"title": "ClickHouse Backend",
"content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
"url": "/archives/v1.3.0/getting-started/clickhouse-backend/#clickhouse-backend",
"relUrl": "/archives/v1.3.0/getting-started/clickhouse-backend/#clickhouse-backend"
- },"412": {
+ },"414": {
"doc": "Getting start with ClickHouse Backend",
"title": "Getting start with ClickHouse Backend",
"content": " ",
"url": "/archives/v1.3.0/getting-started/clickhouse-backend/",
"relUrl": "/archives/v1.3.0/getting-started/clickhouse-backend/"
- },"413": {
+ },"415": {
"doc": "Getting Start with Velox Backend",
"title": "Supported Version",
"content": "| Type | Version | . | Spark | 3.2.2, 3.3.1, 3.4.2, 3.5.1 | . | OS | Ubuntu20.04/22.04, Centos7/8 | . | jdk | openjdk8/jdk17 | . | scala | 2.12 | . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#supported-version",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#supported-version"
- },"414": {
+ },"416": {
"doc": "Getting Start with Velox Backend",
"title": "Prerequisite",
"content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos7/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.2 and 3.5.1. We need to set up the JAVA_HOME env. Currently, Gluten supports java 8 and java 17. For x86_64 . ## make sure jdk8 is used export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#prerequisite",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#prerequisite"
- },"415": {
+ },"417": {
"doc": "Getting Start with Velox Backend",
"title": "Build Gluten with Velox Backend",
"content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . First time build for all supported spark versions./dev/buildbundle-veloxbe.sh . After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and setting up build dependencies./dev/buildbundle-veloxbe.sh --enable_ep [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-velox-backend",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-velox-backend"
- },"416": {
+ },"418": {
"doc": "Getting Start with Velox Backend",
"title": "Dependency library deployment",
"content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#dependency-library-deployment",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#dependency-library-deployment"
- },"417": {
+ },"419": {
"doc": "Getting Start with Velox Backend",
"title": "HDFS support",
"content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#hdfs-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#hdfs-support"
- },"418": {
+ },"420": {
"doc": "Getting Start with Velox Backend",
"title": "Azure Blob File System (ABFS) support",
"content": "Velox supports ABFS with the open source Azure SDK for C++ and Gluten uses the Velox ABFS connector to connect with ABFS. The build option for ABFS (enable_abfs) must be set to enable this feature as listed below. cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_abfs=ON . Please refer Velox ABFS part for more detailed configurations. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#azure-blob-file-system-abfs-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#azure-blob-file-system-abfs-support"
- },"419": {
+ },"421": {
"doc": "Getting Start with Velox Backend",
"title": "AWS S3 support",
"content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#aws-s3-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#aws-s3-support"
- },"420": {
+ },"422": {
"doc": "Getting Start with Velox Backend",
"title": "Celeborn support",
"content": "Gluten with velox backend supports Celeborn as remote shuffle service. Currently, the supported Celeborn versions are 0.3.x, 0.4.x and 0.5.0. Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable celeborn profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests . Then add the Gluten and Spa [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#celeborn-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#celeborn-support"
- },"421": {
+ },"423": {
"doc": "Getting Start with Velox Backend",
"title": "Uniffle support",
"content": "Uniffle with velox backend supports Uniffle as remote shuffle service. Currently, the supported Uniffle versions are 0.8.0. First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with uniffle. When compiling the Gluten Java module, it’s required to enable uniffle profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests . Then add the Uniffle and Spark Celeborn Client packages to your Spark application’s classpath(usu [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#uniffle-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#uniffle-support"
- },"422": {
+ },"424": {
"doc": "Getting Start with Velox Backend",
"title": "DeltaLake Support",
"content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta-XX-SNAPSHOT.jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with vel [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#deltalake-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#deltalake-support"
- },"423": {
+ },"425": {
"doc": "Getting Start with Velox Backend",
"title": "Iceberg Support",
"content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan’s fallback. After the two steps, you can query [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#iceberg-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#iceberg-support"
- },"424": {
+ },"426": {
"doc": "Getting Start with Velox Backend",
"title": "Coverage",
"content": "Spark3.3 has 387 functions in total. ~240 are commonly used. To get the support status of all Spark built-in functions, please refer to Velox Backend’s Supported Operators & Functions. Velox doesn’t support ANSI mode), so as Gluten. Once ANSI mode is enabled in Spark config, Gluten will fallback to Vanilla Spark. To identify what can be offloaded in a query and detailed fallback reasons, user can follow below steps to retrieve corresponding logs. 1) Enable Gluten by p [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#coverage",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#coverage"
- },"425": {
+ },"427": {
"doc": "Getting Start with Velox Backend",
"title": "Spill (Experimental)",
"content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#spill-experimental",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#spill-experimental"
- },"426": {
+ },"428": {
"doc": "Getting Start with Velox Backend",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": "Please check the VeloxNativeUDF.md for more detailed usage and configurations. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"427": {
+ },"429": {
"doc": "Getting Start with Velox Backend",
"title": "High-Bandwidth Memory (HBM) support",
"content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#high-bandwidth-memory-hbm-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#high-bandwidth-memory-hbm-support"
- },"428": {
+ },"430": {
"doc": "Getting Start with Velox Backend",
"title": "Build Gluten with HBM",
"content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.2 and 3.5.1./dev/buildbundle-veloxbe.sh -- [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-hbm",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-hbm"
- },"429": {
+ },"431": {
"doc": "Getting Start with Velox Backend",
"title": "Configure and enable HBM in Spark Application",
"content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application"
- },"430": {
+ },"432": {
"doc": "Getting Start with Velox Backend",
"title": "Intel® QuickAssist Technology (QAT) support",
"content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#intel-quickassist-technology-qat-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#intel-quickassist-technology-qat-support"
- },"431": {
+ },"433": {
"doc": "Getting Start with Velox Backend",
"title": "Software Requirements",
"content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#software-requirements",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#software-requirements"
- },"432": {
+ },"434": {
"doc": "Getting Start with Velox Backend",
"title": "Build Gluten with QAT",
"content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-qat",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-qat"
- },"433": {
+ },"435": {
"doc": "Getting Start with Velox Backend",
"title": "Enable QAT with Gzip/Zstd for shuffle compression",
"content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression"
- },"434": {
+ },"436": {
"doc": "Getting Start with Velox Backend",
"title": "QAT driver references",
"content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#qat-driver-references",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#qat-driver-references"
- },"435": {
+ },"437": {
"doc": "Getting Start with Velox Backend",
"title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
"content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support"
- },"436": {
+ },"438": {
"doc": "Getting Start with Velox Backend",
"title": "Build Gluten with IAA",
"content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-iaa",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-iaa"
- },"437": {
+ },"439": {
"doc": "Getting Start with Velox Backend",
"title": "Enable IAA with Gzip Compression for shuffle compression",
"content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression"
- },"438": {
+ },"440": {
"doc": "Getting Start with Velox Backend",
"title": "IAA references",
"content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#iaa-references",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#iaa-references"
- },"439": {
+ },"441": {
"doc": "Getting Start with Velox Backend",
"title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
"content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
- },"440": {
+ },"442": {
"doc": "Getting Start with Velox Backend",
"title": "Data preparation",
"content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#data-preparation",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#data-preparation"
- },"441": {
+ },"443": {
"doc": "Getting Start with Velox Backend",
"title": "Submit the Spark SQL job",
"content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#submit-the-spark-sql-job",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#submit-the-spark-sql-job"
- },"442": {
+ },"444": {
"doc": "Getting Start with Velox Backend",
"title": "Result",
"content": "wholestagetransformer indicates that the offloading works. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#result",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#result"
- },"443": {
+ },"445": {
"doc": "Getting Start with Velox Backend",
"title": "Performance",
"content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#performance",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#performance"
- },"444": {
+ },"446": {
"doc": "Getting Start with Velox Backend",
"title": "External reference setup",
"content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#external-reference-setup",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#external-reference-setup"
- },"445": {
+ },"447": {
"doc": "Getting Start with Velox Backend",
"title": "Gluten UI",
"content": " ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#gluten-ui",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#gluten-ui"
- },"446": {
+ },"448": {
"doc": "Getting Start with Velox Backend",
"title": "Gluten event",
"content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#gluten-event",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#gluten-event"
- },"447": {
+ },"449": {
"doc": "Getting Start with Velox Backend",
"title": "SQL tab",
"content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#sql-tab",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#sql-tab"
- },"448": {
+ },"450": {
"doc": "Getting Start with Velox Backend",
"title": "History server",
"content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#history-server",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#history-server"
- },"449": {
+ },"451": {
"doc": "Getting Start with Velox Backend",
"title": "Native plan string",
"content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-string",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-string"
- },"450": {
+ },"452": {
"doc": "Getting Start with Velox Backend",
"title": "Native plan with stats",
"content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-with-stats",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-with-stats"
- },"451": {
+ },"453": {
"doc": "Getting Start with Velox Backend",
"title": "Gluten Implicits",
"content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#gluten-implicits",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#gluten-implicits"
- },"452": {
+ },"454": {
"doc": "Getting Start with Velox Backend",
"title": "Getting Start with Velox Backend",
"content": " ",
"url": "/archives/v1.3.0/getting-started/velox-backend/",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/"
- },"453": {
+ },"455": {
"doc": "Using ABFS with Gluten",
"title": "Working with ABFS",
"content": " ",
"url": "/archives/v1.3.0/getting-started/abfs/#working-with-abfs",
"relUrl": "/archives/v1.3.0/getting-started/abfs/#working-with-abfs"
- },"454": {
+ },"456": {
"doc": "Using ABFS with Gluten",
"title": "Configuring ABFS Access Token",
"content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
"url": "/archives/v1.3.0/getting-started/abfs/#configuring-abfs-access-token",
"relUrl": "/archives/v1.3.0/getting-started/abfs/#configuring-abfs-access-token"
- },"455": {
+ },"457": {
"doc": "Using ABFS with Gluten",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/archives/v1.3.0/getting-started/abfs/#local-caching-support",
"relUrl": "/archives/v1.3.0/getting-started/abfs/#local-caching-support"
- },"456": {
+ },"458": {
"doc": "Using ABFS with Gluten",
"title": "Using ABFS with Gluten",
"content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
"url": "/archives/v1.3.0/getting-started/abfs/",
"relUrl": "/archives/v1.3.0/getting-started/abfs/"
- },"457": {
+ },"459": {
"doc": "Using GCS with Gluten",
"title": "Working with GCS",
"content": " ",
"url": "/archives/v1.3.0/getting-started/gcs/#working-with-gcs",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#working-with-gcs"
- },"458": {
+ },"460": {
"doc": "Using GCS with Gluten",
"title": "Installing the gcloud CLI",
"content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
"url": "/archives/v1.3.0/getting-started/gcs/#installing-the-gcloud-cli",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#installing-the-gcloud-cli"
- },"459": {
+ },"461": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS using a user account",
"content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-user-account",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-user-account"
- },"460": {
+ },"462": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS using a credential file",
"content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-credential-file",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-credential-file"
- },"461": {
+ },"463": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS endpoints",
"content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-endpoints",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-endpoints"
- },"462": {
+ },"464": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS max retry count",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-count",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-count"
- },"463": {
+ },"465": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS max retry time",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retring (10s, 1m, etc). ",
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-time",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-time"
- },"464": {
+ },"466": {
"doc": "Using GCS with Gluten",
"title": "Using GCS with Gluten",
"content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.3.0/getting-started/gcs/",
"relUrl": "/archives/v1.3.0/getting-started/gcs/"
- },"465": {
+ },"467": {
"doc": "Velox Local Caching",
"title": "Velox Local Caching",
"content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
"url": "/archives/v1.3.0/getting-started/localcache/",
"relUrl": "/archives/v1.3.0/getting-started/localcache/"
- },"466": {
+ },"468": {
"doc": "Using S3 with Gluten",
"title": "Working with S3",
"content": " ",
"url": "/archives/v1.3.0/getting-started/s3/#working-with-s3",
"relUrl": "/archives/v1.3.0/getting-started/s3/#working-with-s3"
- },"467": {
+ },"469": {
"doc": "Using S3 with Gluten",
"title": "Configuring S3 endpoint",
"content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
"url": "/archives/v1.3.0/getting-started/s3/#configuring-s3-endpoint",
"relUrl": "/archives/v1.3.0/getting-started/s3/#configuring-s3-endpoint"
- },"468": {
+ },"470": {
"doc": "Using S3 with Gluten",
"title": "Configuring S3 instance credentials",
"content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
"url": "/archives/v1.3.0/getting-started/s3/#configuring-s3-instance-credentials",
"relUrl": "/archives/v1.3.0/getting-started/s3/#configuring-s3-instance-credentials"
- },"469": {
+ },"471": {
"doc": "Using S3 with Gluten",
"title": "Configuring S3 IAM roles",
"content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
"url": "/archives/v1.3.0/getting-started/s3/#configuring-s3-iam-roles",
"relUrl": "/archives/v1.3.0/getting-started/s3/#configuring-s3-iam-roles"
- },"470": {
+ },"472": {
"doc": "Using S3 with Gluten",
"title": "Other authentatication methods are not supported yet",
"content": ". | AWS temporary credential | . ",
"url": "/archives/v1.3.0/getting-started/s3/#other-authentatication-methods-are-not-supported-yet",
"relUrl": "/archives/v1.3.0/getting-started/s3/#other-authentatication-methods-are-not-supported-yet"
- },"471": {
+ },"473": {
"doc": "Using S3 with Gluten",
"title": "Log granularity of AWS C++ SDK in velox",
"content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
"url": "/archives/v1.3.0/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox",
"relUrl": "/archives/v1.3.0/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox"
- },"472": {
+ },"474": {
"doc": "Using S3 with Gluten",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/archives/v1.3.0/getting-started/s3/#local-caching-support",
"relUrl": "/archives/v1.3.0/getting-started/s3/#local-caching-support"
- },"473": {
+ },"475": {
"doc": "Using S3 with Gluten",
"title": "Using S3 with Gluten",
"content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.3.0/getting-started/s3/",
"relUrl": "/archives/v1.3.0/getting-started/s3/"
- },"474": {
+ },"476": {
"doc": "Build Parameters for Velox Backend",
"title": "Build Parameters",
"content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | OFF | . | build_proto [...]
"url": "/archives/v1.3.0/getting-started/build-guide/#build-parameters",
"relUrl": "/archives/v1.3.0/getting-started/build-guide/#build-parameters"
- },"475": {
+ },"477": {
"doc": "Build Parameters for Velox Backend",
"title": "Gluten Jar for Deployment",
"content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . | 3.5.1 | 3.5 | 2.12 | . ",
"url": "/archives/v1.3.0/getting-started/build-guide/#gluten-jar-for-deployment",
"relUrl": "/archives/v1.3.0/getting-started/build-guide/#gluten-jar-for-deployment"
- },"476": {
+ },"478": {
"doc": "Build Parameters for Velox Backend",
"title": "Build Parameters for Velox Backend",
"content": " ",
"url": "/archives/v1.3.0/getting-started/build-guide/",
"relUrl": "/archives/v1.3.0/getting-started/build-guide/"
- },"477": {
+ },"479": {
"doc": "Getting-Started",
"title": "Getting Started with Gluten for Apache Spark",
"content": " ",
"url": "/archives/v1.3.0/getting-started#getting-started-with-gluten-for-apache-spark",
"relUrl": "/archives/v1.3.0/getting-started#getting-started-with-gluten-for-apache-spark"
- },"478": {
+ },"480": {
"doc": "Getting-Started",
"title": "Getting-Started",
"content": " ",
"url": "/archives/v1.3.0/getting-started",
"relUrl": "/archives/v1.3.0/getting-started"
- },"479": {
+ },"481": {
"doc": "v1.3.0",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.3.0/#gluten-documents-by-version",
"relUrl": "/archives/v1.3.0/#gluten-documents-by-version"
- },"480": {
+ },"482": {
"doc": "v1.3.0",
"title": "v1.3.0",
"content": " ",
"url": "/archives/v1.3.0/",
"relUrl": "/archives/v1.3.0/"
- },"481": {
+ },"483": {
"doc": "Velox Backend",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.3.0/velox-backend#gluten-documents-by-version",
"relUrl": "/archives/v1.3.0/velox-backend#gluten-documents-by-version"
- },"482": {
+ },"484": {
"doc": "Velox Backend",
"title": "Velox Backend",
"content": " ",
"url": "/archives/v1.3.0/velox-backend",
"relUrl": "/archives/v1.3.0/velox-backend"
- },"483": {
+ },"485": {
"doc": "Limitations",
"title": "Velox Backend Limitations",
"content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
"url": "/archives/v1.3.0/velox-backend/limitations/#velox-backend-limitations",
"relUrl": "/archives/v1.3.0/velox-backend/limitations/#velox-backend-limitations"
- },"484": {
+ },"486": {
"doc": "Limitations",
"title": "Limitations",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/limitations/",
"relUrl": "/archives/v1.3.0/velox-backend/limitations/"
- },"485": {
+ },"487": {
"doc": "Supported Operators & Functions",
"title": "The Operators and Functions Support Progress",
"content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. The total number of fun [...]
"url": "/archives/v1.3.0/velox-backend/support/#the-operators-and-functions-support-progress",
"relUrl": "/archives/v1.3.0/velox-backend/support/#the-operators-and-functions-support-progress"
- },"486": {
+ },"488": {
"doc": "Supported Operators & Functions",
"title": "Supported Operators & Functions",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/support/",
"relUrl": "/archives/v1.3.0/velox-backend/support/"
- },"487": {
+ },"489": {
"doc": "Troubleshooting",
"title": "Velox Backend Troubleshooting",
"content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
"url": "/archives/v1.3.0/velox-backend/troubleshooting/#velox-backend-troubleshooting",
"relUrl": "/archives/v1.3.0/velox-backend/troubleshooting/#velox-backend-troubleshooting"
- },"488": {
+ },"490": {
"doc": "Troubleshooting",
"title": "Troubleshooting",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/troubleshooting/",
"relUrl": "/archives/v1.3.0/velox-backend/troubleshooting/"
- },"489": {
+ },"491": {
"doc": "Velox UDF",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"490": {
+ },"492": {
"doc": "Velox UDF",
"title": "Introduction",
"content": "Velox backend supports User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
"url": "/archives/v1.3.0/velox-backend/udf/#introduction",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#introduction"
- },"491": {
+ },"493": {
"doc": "Velox UDF",
"title": "Create and Build UDF/UDAF library",
"content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
"url": "/archives/v1.3.0/velox-backend/udf/#create-and-build-udfudaf-library",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#create-and-build-udfudaf-library"
- },"492": {
+ },"494": {
"doc": "Velox UDF",
"title": "Using UDF/UDAF in Gluten",
"content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the library paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
"url": "/archives/v1.3.0/velox-backend/udf/#using-udfudaf-in-gluten",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#using-udfudaf-in-gluten"
- },"493": {
+ },"495": {
"doc": "Velox UDF",
"title": "Try the example",
"content": "We provided Velox UDF examples in file MyUDF.cc and UDAF examples in file MyUDAF.cc. You need to build the gluten project with --build_example=ON to get the example libraries./dev/buildbundle-veloxbe.sh --build_examples=ON . Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ . Start spark-shell or spark-sql with below configuration . # Use the `--files` option to upload a library and configure its relative path --files /path/to/glute [...]
"url": "/archives/v1.3.0/velox-backend/udf/#try-the-example",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#try-the-example"
- },"494": {
+ },"496": {
"doc": "Velox UDF",
"title": "Configurations",
"content": "| Parameters | Description | . | spark.gluten.sql.columnar.backend.velox.udfLibraryPaths | Path to the udf/udaf libraries. | . | spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths | Path to the udf/udaf libraries on driver node. Only applicable on yarn-client mode. | . | spark.gluten.sql.columnar.backend.velox.udfAllowTypeConversion | Whether to inject possible cast to convert mismatched data types from input to one registered signatures. | . ",
"url": "/archives/v1.3.0/velox-backend/udf/#configurations",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#configurations"
- },"495": {
+ },"497": {
"doc": "Velox UDF",
"title": "Pandas UDFs (a.k.a. Vectorized UDFs)",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs"
- },"496": {
+ },"498": {
"doc": "Velox UDF",
"title": "Introduction",
"content": "Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer doc. ",
"url": "/archives/v1.3.0/velox-backend/udf/#introduction-1",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#introduction-1"
- },"497": {
+ },"499": {
"doc": "Velox UDF",
"title": "Using Pandas UDFs in Gluten with Velox Backend",
"content": "Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: . pip3 install pyspark==$SPARK_VERSION cython pip3 install pandas pyarrow . Gluten provides a config to control enable ColumnarArrowEvalPython or not, with true as defalt. spark.gluten.sql.columnar.arrowUdf . Then take following PySpark code for example: . from pyspark.sql.functions import pandas_udf, PandasUDFType import pyspark.sql.functions as [...]
"url": "/archives/v1.3.0/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend"
- },"498": {
+ },"500": {
"doc": "Velox UDF",
"title": "Velox UDF",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/udf/",
"relUrl": "/archives/v1.3.0/velox-backend/udf/"
- },"499": {
+ },"501": {
"doc": "Configuration",
"title": "Spark Configurations for Gluten Plugin",
"content": "There are many configurations could impact the Gluten Plugin performance and can be fine-tuned in Spark. You can add these configurations into spark-defaults.conf to enable or disable the setting. ",
"url": "/docs/v1.4.0/Configuration.html#spark-configurations-for-gluten-plugin",
"relUrl": "/docs/v1.4.0/Configuration.html#spark-configurations-for-gluten-plugin"
- },"500": {
+ },"502": {
"doc": "Configuration",
"title": "Spark parameters",
"content": "| Parameters | Description | Recommend Setting | . | spark.driver.extraClassPath | Gluten Plugin jar file to prepend to the classpath of the driver. | /path/to/jar_file | . | spark.executor.extraClassPath | Gluten Plugin jar file to prepend to the classpath of executors. | /path/to/jar_file | . | spark.executor.memory | Amount of memory to use per executor process, in the same format as JVM memory strings with a size unit suffix (“k”, “m”, “g” or “t”) (e.g. 512m, 2g). | [...]
"url": "/docs/v1.4.0/Configuration.html#spark-parameters",
"relUrl": "/docs/v1.4.0/Configuration.html#spark-parameters"
- },"501": {
+ },"503": {
"doc": "Configuration",
"title": "Velox Parameters",
"content": "The following configurations are related to Velox settings. | Parameters | Description | Recommend Setting | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The m [...]
"url": "/docs/v1.4.0/Configuration.html#velox-parameters",
"relUrl": "/docs/v1.4.0/Configuration.html#velox-parameters"
- },"502": {
+ },"504": {
"doc": "Configuration",
"title": "Configuration",
"content": " ",
"url": "/docs/v1.4.0/Configuration.html",
"relUrl": "/docs/v1.4.0/Configuration.html"
- },"503": {
+ },"505": {
"doc": "CppCodingStyle",
"title": "Gluten CPP Core Guidelines",
"content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#gluten-cpp-core-guidelines",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#gluten-cpp-core-guidelines"
- },"504": {
+ },"506": {
"doc": "CppCodingStyle",
"title": "Philosophy",
"content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#philosophy",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#philosophy"
- },"505": {
+ },"507": {
"doc": "CppCodingStyle",
"title": "Code Formatting",
"content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). | Always ensure your code is compatible with clang-format-15 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . To format cmake files like CMakeLists.txt & *.cmake, cmake-format is required to be installed. Here is an example. apt install python3-pip -y pip3 install --use [...]
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#code-formatting",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#code-formatting"
- },"506": {
+ },"508": {
"doc": "CppCodingStyle",
"title": "Naming Conventions",
"content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#naming-conventions",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#naming-conventions"
- },"507": {
+ },"509": {
"doc": "CppCodingStyle",
"title": "Designs",
"content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#designs",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#designs"
- },"508": {
+ },"510": {
"doc": "CppCodingStyle",
"title": "Source File & Header File",
"content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#source-file--header-file",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#source-file--header-file"
- },"509": {
+ },"511": {
"doc": "CppCodingStyle",
"title": "Class",
"content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#class",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#class"
- },"510": {
+ },"512": {
"doc": "CppCodingStyle",
"title": "Function",
"content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#function",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#function"
- },"511": {
+ },"513": {
"doc": "CppCodingStyle",
"title": "Variable",
"content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#variable",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#variable"
- },"512": {
+ },"514": {
"doc": "CppCodingStyle",
"title": "Constant",
"content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#constant",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#constant"
- },"513": {
+ },"515": {
"doc": "CppCodingStyle",
"title": "Macro",
"content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#macro",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#macro"
- },"514": {
+ },"516": {
"doc": "CppCodingStyle",
"title": "Namespace",
"content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#namespace",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#namespace"
- },"515": {
+ },"517": {
"doc": "CppCodingStyle",
"title": "Resource Management",
"content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#resource-management",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#resource-management"
- },"516": {
+ },"518": {
"doc": "CppCodingStyle",
"title": "Exception",
"content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#exception",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#exception"
- },"517": {
+ },"519": {
"doc": "CppCodingStyle",
"title": "Code Comment",
"content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#code-comment",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#code-comment"
- },"518": {
+ },"520": {
"doc": "CppCodingStyle",
"title": "References",
"content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html#references",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html#references"
- },"519": {
+ },"521": {
"doc": "CppCodingStyle",
"title": "CppCodingStyle",
"content": " ",
"url": "/docs/v1.4.0/developers/CppCodingStyle.html",
"relUrl": "/docs/v1.4.0/developers/CppCodingStyle.html"
- },"520": {
+ },"522": {
"doc": "HowTo",
"title": "How to understand the key work of Gluten?",
"content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsible for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the direct [...]
"url": "/docs/v1.4.0/developers/HowTo.html#how-to-understand-the-key-work-of-gluten",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#how-to-understand-the-key-work-of-gluten"
- },"521": {
+ },"523": {
"doc": "HowTo",
"title": "How to debug in Gluten?",
"content": " ",
"url": "/docs/v1.4.0/developers/HowTo.html#how-to-debug-in-gluten",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#how-to-debug-in-gluten"
- },"522": {
+ },"524": {
"doc": "HowTo",
"title": "1 How to debug C++",
"content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | Build Velox and Gluten CPP: | . ${GLUTEN_HOME}/dev/builddeps-velox [...]
"url": "/docs/v1.4.0/developers/HowTo.html#1-how-to-debug-c",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#1-how-to-debug-c"
- },"523": {
+ },"525": {
"doc": "HowTo",
"title": "2 How to debug plan validation process",
"content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in ${GLUTEN_HOME}/cpp/build/velox/benchmarks/. | R [...]
"url": "/docs/v1.4.0/developers/HowTo.html#2-how-to-debug-plan-validation-process",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#2-how-to-debug-plan-validation-process"
- },"524": {
+ },"526": {
"doc": "HowTo",
"title": "3 How to debug Java/Scala",
"content": "To debug some runtime issues in Scala/Java, we recommend developers to use Intellij remote debug, see tutorial link. According to your setting for Intellij remote debug, please set SPARK_SUBMIT_OPTS in the environment where spark-submit is executed. See the below example. export SPARK_SUBMIT_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8008 . To run a Scala/Java test class, you can use the below mvn command (take Velox backend as example), which is h [...]
"url": "/docs/v1.4.0/developers/HowTo.html#3-how-to-debug-javascala",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#3-how-to-debug-javascala"
- },"525": {
+ },"527": {
"doc": "HowTo",
"title": "4 How to debug with core-dump",
"content": "wait to complete . cd the_directory_of_core_file_generated gdb ${GLUTEN_HOME}/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
"url": "/docs/v1.4.0/developers/HowTo.html#4-how-to-debug-with-core-dump",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#4-how-to-debug-with-core-dump"
- },"526": {
+ },"528": {
"doc": "HowTo",
"title": "How to use jemalloc for Gluten native engine",
"content": "Currently, we have no dedicated memory allocator implemented by jemalloc. User can set environment variable LD_PRELOAD for lib jemalloc to let it override the corresponding C standard functions entirely. It may help alleviate OOM issues. spark.executorEnv.LD_PREALOD=/path/to/libjemalloc.so . ",
"url": "/docs/v1.4.0/developers/HowTo.html#how-to-use-jemalloc-for-gluten-native-engine",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#how-to-use-jemalloc-for-gluten-native-engine"
- },"527": {
+ },"529": {
"doc": "HowTo",
"title": "How to run TPC-H on Velox backend",
"content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of ${GLUTEN_HOME}/backends-velox/workload/tpch. The file README.md under ${GLUTEN_HOME}/backends-velox/workload/tpch offers some useful help, but it’s still not enough and exact. One way of run TPC-H test is to run velox-be by workflow, you can refer to velox_backend.yml . Here we will explain how to run TPC-H on Velox backend with the Parquet file format. | First, pre [...]
"url": "/docs/v1.4.0/developers/HowTo.html#how-to-run-tpc-h-on-velox-backend",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#how-to-run-tpc-h-on-velox-backend"
- },"528": {
+ },"530": {
"doc": "HowTo",
"title": "How to run TPC-DS",
"content": "wait to add . ",
"url": "/docs/v1.4.0/developers/HowTo.html#how-to-run-tpc-ds",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#how-to-run-tpc-ds"
- },"529": {
+ },"531": {
"doc": "HowTo",
"title": "How to track the memory exhaust problem",
"content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.memory.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of mem [...]
"url": "/docs/v1.4.0/developers/HowTo.html#how-to-track-the-memory-exhaust-problem",
"relUrl": "/docs/v1.4.0/developers/HowTo.html#how-to-track-the-memory-exhaust-problem"
- },"530": {
+ },"532": {
"doc": "HowTo",
"title": "HowTo",
"content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. When you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
"url": "/docs/v1.4.0/developers/HowTo.html",
"relUrl": "/docs/v1.4.0/developers/HowTo.html"
- },"531": {
+ },"533": {
"doc": "MicroBenchmarks",
"title": "Generate Micro Benchmarks for Velox Backend",
"content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#generate-micro-benchmarks-for-velox-backend",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#generate-micro-benchmarks-for-velox-backend"
- },"532": {
+ },"534": {
"doc": "MicroBenchmarks",
"title": "Try the example",
"content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3. mvn test -Pspark-3.2 -Pbackends-velox -pl backends-velox [...]
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#try-the-example",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#try-the-example"
- },"533": {
+ },"535": {
"doc": "MicroBenchmarks",
"title": "Generate Substrait plan and input for any query",
"content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#generate-substrait-plan-and-input-for-any-query",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#generate-substrait-plan-and-input-for-any-query"
- },"534": {
+ },"536": {
"doc": "MicroBenchmarks",
"title": "Save output to parquet for analysis",
"content": "You can save the output to a parquet file via --save-output <output> . Note: 1. This option cannot be used together with --with-shuffle. 2. This option cannot be used for write tasks. Please refer to section Simulate write tasks for more details. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --save-output /absolute_path/to/result.parquet . ",
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#save-output-to-parquet-for-analysis",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#save-output-to-parquet-for-analysis"
- },"535": {
+ },"537": {
"doc": "MicroBenchmarks",
"title": "Add shuffle write process",
"content": "You can add the shuffle write process at the end of the pipeline via --with-shuffle . Note: 1. This option cannot be used together with --save-output. 2. This option cannot be used for write tasks. Please refer to section Simulate write tasks for more details. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . Developers can leverag [...]
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#add-shuffle-write-process",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#add-shuffle-write-process"
- },"536": {
+ },"538": {
"doc": "MicroBenchmarks",
"title": "Enable debug mode",
"content": "spark.gluten.sql.debug(debug mode) is set to false by default thereby the google glog levels are limited to only print WARNING or higher severity logs. Unless spark.gluten.sql.debug is set in the INI file via --conf, the logging behavior is same as debug mode off. Developers can use --debug-mode command line flag to turn on debug mode when needed, and set verbosity/severity level via command line flags --v and --minloglevel. Note that constructing and deconstructing log s [...]
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#enable-debug-mode",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#enable-debug-mode"
- },"537": {
+ },"539": {
"doc": "MicroBenchmarks",
"title": "Enable HDFS support",
"content": "After enabling the dynamic loading of libhdfs.so at runtime to support HDFS, if you run the benchmark with an HDFS file, you need to set the classpath for Hadoop. You can do this by running . export CLASSPATH=`$HADOOP_HOME/bin/hdfs classpath --glob` . Otherwise, the HDFS connection will fail. If you have replaced ${HADOOP_HOME}/lib/native/libhdfs.so with libhdfs3.so, there is no need to set the CLASSPATH. ",
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#enable-hdfs-support",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#enable-hdfs-support"
- },"538": {
+ },"540": {
"doc": "MicroBenchmarks",
"title": "Simulate write tasks",
"content": "The last operator for a write task is a file write operator, and the output from Velox pipeline only contains several columns of statistics data. Therefore, specifying options --with-shuffle and --save-output does not take effect. You can specify the output path for the writer via --write-path option. Default is /tmp. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --write-p [...]
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#simulate-write-tasks",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#simulate-write-tasks"
- },"539": {
+ },"541": {
"doc": "MicroBenchmarks",
"title": "Simulate task spilling",
"content": "You can simulate task spilling by specify a memory hard limit from --memory_limit. By default, spilled files are written to the /tmp directory. To simulate real Gluten workloads, which utilize multiple spill directories, set the environment variable GLUTEN_SPARK_LOCAL_DIRS to a comma-separated string. Please check Simulate Gluten workload with multiple processes and threads for more details. ",
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#simulate-task-spilling",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#simulate-task-spilling"
- },"540": {
+ },"542": {
"doc": "MicroBenchmarks",
"title": "Simulate Gluten workload with multiple processes and threads",
"content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a bare-metal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --pla [...]
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html#simulate-gluten-workload-with-multiple-processes-and-threads",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html#simulate-gluten-workload-with-multiple-processes-and-threads"
- },"541": {
+ },"543": {
"doc": "MicroBenchmarks",
"title": "MicroBenchmarks",
"content": " ",
"url": "/docs/v1.4.0/developers/MicroBenchmarks.html",
"relUrl": "/docs/v1.4.0/developers/MicroBenchmarks.html"
- },"542": {
+ },"544": {
"doc": "NewToGluten",
"title": "Environment",
"content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
"url": "/docs/v1.4.0/developers/NewToGluten.html#environment",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#environment"
- },"543": {
+ },"545": {
"doc": "NewToGluten",
"title": "JDK",
"content": "Currently, Gluten supports JDK 8 for Spark 3.2/3.3/3.4/3.5. For Spark 3.3 and higher versions, Gluten supports JDK 11 and 17. Please note since Spark 4.0, JDK 8 will not be supported. So we recommend Velox backend users to use higher JDK version now to ease the migration for deploying Gluten with Spark-4.0 in the future. And we may probably upgrade Arrow from 15.0.0 to some higher version, which also requires JDK 11 is the minimum version. JDK 8 . Environment Setting . Fo [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#jdk",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#jdk"
- },"544": {
+ },"546": {
"doc": "NewToGluten",
"title": "Maven 3.6.3 or above",
"content": "Maven Download Page And then set the environment setting. ",
"url": "/docs/v1.4.0/developers/NewToGluten.html#maven-363-or-above",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#maven-363-or-above"
- },"545": {
+ },"547": {
"doc": "NewToGluten",
"title": "GCC 11 or above",
"content": " ",
"url": "/docs/v1.4.0/developers/NewToGluten.html#gcc-11-or-above",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#gcc-11-or-above"
- },"546": {
+ },"548": {
"doc": "NewToGluten",
"title": "Compile gluten using debug mode",
"content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#compile-gluten-using-debug-mode",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#compile-gluten-using-debug-mode"
- },"547": {
+ },"549": {
"doc": "NewToGluten",
"title": "Java/scala code development with Intellij",
"content": " ",
"url": "/docs/v1.4.0/developers/NewToGluten.html#javascala-code-development-with-intellij",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#javascala-code-development-with-intellij"
- },"548": {
+ },"550": {
"doc": "NewToGluten",
"title": "Linux IntelliJ local debug",
"content": "Install the Linux IntelliJ version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download IntelliJ Linux community version to Linux server | Start Idea, bash <idea_dir>/idea.sh | . ",
"url": "/docs/v1.4.0/developers/NewToGluten.html#linux-intellij-local-debug",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#linux-intellij-local-debug"
- },"549": {
+ },"551": {
"doc": "NewToGluten",
"title": "Set up gluten project",
"content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
"url": "/docs/v1.4.0/developers/NewToGluten.html#set-up-gluten-project",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#set-up-gluten-project"
- },"550": {
+ },"552": {
"doc": "NewToGluten",
"title": "Java/Scala code style",
"content": "IntelliJ supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See IntelliJ guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.3 -Pspark-ut -Dskip [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#javascala-code-style",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#javascala-code-style"
- },"551": {
+ },"553": {
"doc": "NewToGluten",
"title": "CPP code development with Visual Studio Code",
"content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#cpp-code-development-with-visual-studio-code",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#cpp-code-development-with-visual-studio-code"
- },"552": {
+ },"554": {
"doc": "NewToGluten",
"title": "Usage",
"content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile using debug mode in /build. And we need to compile velox debug mode before, if you have compiled [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#usage",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#usage"
- },"553": {
+ },"555": {
"doc": "NewToGluten",
"title": "Useful notes",
"content": "Do not upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode. Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#useful-notes",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#useful-notes"
- },"554": {
+ },"556": {
"doc": "NewToGluten",
"title": "Debug cpp code with coredump",
"content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#debug-cpp-code-with-coredump",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#debug-cpp-code-with-coredump"
- },"555": {
+ },"557": {
"doc": "NewToGluten",
"title": "Debug cpp with gdb",
"content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#debug-cpp-with-gdb",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#debug-cpp-with-gdb"
- },"556": {
+ },"558": {
"doc": "NewToGluten",
"title": "Debug Memory leak",
"content": " ",
"url": "/docs/v1.4.0/developers/NewToGluten.html#debug-memory-leak",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#debug-memory-leak"
- },"557": {
+ },"559": {
"doc": "NewToGluten",
"title": "Arrow memory allocator leak",
"content": "If you receive error message like . 4/04/18 08:15:38 WARN ArrowBufferAllocators$ArrowBufferAllocatorManager: Detected leaked Arrow allocator [Default], size: 191, process accumulated leaked size: 191... 24/04/18 08:15:38 WARN ArrowBufferAllocators$ArrowBufferAllocatorManager: Leaked allocator stack Allocator(ROOT) 0/191/319/9223372036854775807 (res/actual/peak/limit) . You can open the Arrow allocator debug config by add VP option -Darrow.memory.debug.allocator=true, then [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#arrow-memory-allocator-leak",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#arrow-memory-allocator-leak"
- },"558": {
+ },"560": {
"doc": "NewToGluten",
"title": "CPP code memory leak",
"content": "Sometimes you cannot get the coredump symbols, if you debug memory leak, you can write googletest to use valgrind to detect . apt install valgrind valgrind --leak-check=yes ./exec_backend_test . ",
"url": "/docs/v1.4.0/developers/NewToGluten.html#cpp-code-memory-leak",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#cpp-code-memory-leak"
- },"559": {
+ },"561": {
"doc": "NewToGluten",
"title": "Run TPC-H and TPC-DS",
"content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_backend.yml . ",
"url": "/docs/v1.4.0/developers/NewToGluten.html#run-tpc-h-and-tpc-ds",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#run-tpc-h-and-tpc-ds"
- },"560": {
+ },"562": {
"doc": "NewToGluten",
"title": "Run gluten+velox on clean machine",
"content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=org.apache.gluten.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/apache/incubator-gluten/releases/download/v1.1.1/gluten-velox-bundle-spark3.2_2.12-1.1.1.jar \\ --conf spark.shuffle.manager=org. [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#run-glutenvelox-on-clean-machine",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#run-glutenvelox-on-clean-machine"
- },"561": {
+ },"563": {
"doc": "NewToGluten",
"title": "Check Gluten Approved Spark Plan",
"content": "To make sure we don’t accidentally modify the Gluten and Spark Plan build logic. We introduce new logic in VeloxTPCHSuite to check whether the plan has been changed or not, and this will be triggered when running the unit test. As a result, developers may encounter unit test fail in Github CI or locally, with the following error message: . - TPC-H q5 *** FAILED *** Mismatch for query 5 Actual Plan path: /tmp/tpch-approved-plan/v2-bhj/spark322/5.txt Golden Plan path: /opt/ [...]
"url": "/docs/v1.4.0/developers/NewToGluten.html#check-gluten-approved-spark-plan",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html#check-gluten-approved-spark-plan"
- },"562": {
+ },"564": {
"doc": "NewToGluten",
"title": "NewToGluten",
"content": "Help users to debug and test with gluten. ",
"url": "/docs/v1.4.0/developers/NewToGluten.html",
"relUrl": "/docs/v1.4.0/developers/NewToGluten.html"
- },"563": {
+ },"565": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Profile with gperftools",
"content": "gperftools is a collection of a high-performance multi-threaded malloc() implementation, plus some pretty nifty performance analysis tools, see more: https://github.com/gperftools/gperftools/wiki. ",
"url": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#profile-with-gperftools",
"relUrl": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#profile-with-gperftools"
- },"564": {
+ },"566": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Build and install gperftools",
"content": "Download gperftools from https://github.com/gperftools/gperftools/releases, build and install. wget https://github.com/gperftools/gperftools/releases/download/gperftools<version>/gperftools-<version>.tar.gz tar xzvf gperftools-<version>.tar.gz cd gperftools-<version> ./configure make && make install . Then we can find the tcmalloc libraries in $GPERFTOOLS_HOME/.lib. ",
"url": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#build-and-install-gperftools",
"relUrl": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#build-and-install-gperftools"
- },"565": {
+ },"567": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Run Gluten with gperftools",
"content": "Configure --files or spark.files for Spark. --files /path/to/gperftools/libtcmalloc_and_profiler.so or spark.files /path/to/gperftools/libtcmalloc_and_profiler.so . Use LD_PRELOAD to preload tcmalloc library, and enable heap profile with HEAPPROFILE or cpu profile with CPUPROFILE. Example of enabling heap profile in spark executor: . spark.executorEnv.LD_PRELOAD ./libtcmalloc_and_profiler.so # Specifies dump profile path. ${CONTAINER_ID} is only used to distinguish the re [...]
"url": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#run-gluten-with-gperftools",
"relUrl": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#run-gluten-with-gperftools"
- },"566": {
+ },"568": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Analyze profiling output",
"content": "Prepare the required native libraries. Assume static build is used for Gluten, there is no other shared dependency libs. jar xf gluten-velox-bundle-spark3.5_2.12-centos_7_x86_64-1.2.0.jar relative/path/to/libvelox.so ralative/path/to/libgluten.so mv libvelox.so libgluten.so /path/to/gluten_lib_prefix . Generate a GIF of the analysis result: . # `/usr/bin/java` indicates the program used by running spark executor pprof --show_bytes --gif --lib_prefix=/path/to/gluten_lib_pr [...]
"url": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#analyze-profiling-output",
"relUrl": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#analyze-profiling-output"
- },"567": {
+ },"569": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Profile with jemalloc",
"content": "jemalloc is a general purpose malloc(3) implementation that emphasizes fragmentation avoidance and scalable concurrency support. We can also use it to analyze Gluten performance. Getting Started with jemalloc: https://github.com/jemalloc/jemalloc/wiki/Getting-Started. ",
"url": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#profile-with-jemalloc",
"relUrl": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#profile-with-jemalloc"
- },"568": {
+ },"570": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Build and install jemalloc",
"content": "Download jemalloc from https://github.com/jemalloc/jemalloc/releases, build and install. cd /path/to/jemalloc ./autogen.sh --enable-prof make && make install . Then we can find the jemalloc library in $JEMALLOC_HOME/.lib. ",
"url": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#build-and-install-jemalloc",
"relUrl": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#build-and-install-jemalloc"
- },"569": {
+ },"571": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Run Gluten with jemalloc",
"content": "Configure --files or spark.files for Spark. --files /path/to/jemalloc/libjemalloc.so or spark.files /path/to/jemalloc/libjemalloc.so . Example of enabling heap profile in spark executor: . spark.executorEnv.LD_PRELOAD ./libjemalloc.so spark.executorEnv.MALLOC_CONF prof:true,lg_prof_interval:30,prof_prefix:/tmp/gluten_heap_perf . Finally, profiling files prefixed with /tmp/gluten_heap_perf.${PID} will be generated for each spark executor. ",
"url": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#run-gluten-with-jemalloc",
"relUrl": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#run-gluten-with-jemalloc"
- },"570": {
+ },"572": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Analyze profiling output",
"content": "Prepare the required native libraries. Assume static build is used for Gluten, so there is no other shared dependency libs. jar xf gluten-velox-bundle-spark3.5_2.12-centos_7_x86_64-1.2.0.jar relative/path/to/libvelox.so relative/path/to/libgluten.so mv libvelox.so libgluten.so /path/to/gluten_lib_prefix . Generate a GIF of the analysis result: . # `/usr/bin/java` indicates the program used by running spark executor jeprof --show_bytes --gif --lib_prefix=/path/to/gluten_li [...]
"url": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#analyze-profiling-output-1",
"relUrl": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html#analyze-profiling-output-1"
- },"571": {
+ },"573": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "ProfileMemoryOfGlutenWithVelox",
"content": "Gluten offloads most of Spark SQL execution to native engine. We can use gperftools or jemalloc to analyze the offheap memory and cpu profile. ",
"url": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html",
"relUrl": "/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html"
- },"572": {
+ },"574": {
"doc": "QueryTrace",
"title": "Background",
"content": "Currently, we have MicroBenchmarks to profile the Velox plan execution in stage level, now we have query trace replayer to specify plan node id to profile in operator level. We cannot profile operator level directly because ValueStreamNode cannot serialize and deserialize, so we should generate benchmark first, and then enable query trace in benchmark which replaces the ValueStreamNode to ValuesNode. Relevant link: https://facebookincubator.github.io/velox/develop/debuggi [...]
"url": "/docs/v1.4.0/developers/QueryTrace.html#background",
"relUrl": "/docs/v1.4.0/developers/QueryTrace.html#background"
- },"573": {
+ },"575": {
"doc": "QueryTrace",
"title": "Usage",
"content": "Firstly, we should use MicroBenchmark to generate the input parquet data file and substrait json plan. /tmp/saveDir ├── conf_35_0.ini ├── data_35_0_0.parquet ├── plan_35_0.json └── split_35_0_0.json . Secondly, execute the benchmark to get the task id and query trace node id, for example, the task id is Gluten_Stage_0_TID_0_VTID_0 and we will profile partial aggregation of node id 7. W0113 15:30:18.903225 2524110 GenericBenchmark.cc:251] Setting CPU for thread 0 to 0 I011 [...]
"url": "/docs/v1.4.0/developers/QueryTrace.html#usage",
"relUrl": "/docs/v1.4.0/developers/QueryTrace.html#usage"
- },"574": {
+ },"576": {
"doc": "QueryTrace",
"title": "QueryTrace",
"content": " ",
"url": "/docs/v1.4.0/developers/QueryTrace.html",
"relUrl": "/docs/v1.4.0/developers/QueryTrace.html"
- },"575": {
+ },"577": {
"doc": "SubstraitModifications",
"title": "Substrait Modifications in Gluten",
"content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
"url": "/docs/v1.4.0/developers/SubstraitModifications.html#substrait-modifications-in-gluten",
"relUrl": "/docs/v1.4.0/developers/SubstraitModifications.html#substrait-modifications-in-gluten"
- },"576": {
+ },"578": {
"doc": "SubstraitModifications",
"title": "Modifications to algebra.proto",
"content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
"url": "/docs/v1.4.0/developers/SubstraitModifications.html#modifications-to-algebraproto",
"relUrl": "/docs/v1.4.0/developers/SubstraitModifications.html#modifications-to-algebraproto"
- },"577": {
+ },"579": {
"doc": "SubstraitModifications",
"title": "Modifications to type.proto",
"content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
"url": "/docs/v1.4.0/developers/SubstraitModifications.html#modifications-to-typeproto",
"relUrl": "/docs/v1.4.0/developers/SubstraitModifications.html#modifications-to-typeproto"
- },"578": {
+ },"580": {
"doc": "SubstraitModifications",
"title": "SubstraitModifications",
"content": " ",
"url": "/docs/v1.4.0/developers/SubstraitModifications.html",
"relUrl": "/docs/v1.4.0/developers/SubstraitModifications.html"
- },"579": {
+ },"581": {
"doc": "UsingGperftoolsInCH",
"title": "Install gperftools",
"content": "Install gperftools as described in https://github.com/gperftools/gperftools. We get the library and the command line tools. ",
"url": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html#install-gperftools",
"relUrl": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html#install-gperftools"
- },"580": {
+ },"582": {
"doc": "UsingGperftoolsInCH",
"title": "Compiler libch.so",
"content": "Disable jemalloc -DENABLE_JEMALLOC=OFF in cpp-ch/CMakeLists.txt, and recompile libch.so. ",
"url": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html#compiler-libchso",
"relUrl": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html#compiler-libchso"
- },"581": {
+ },"583": {
"doc": "UsingGperftoolsInCH",
"title": "Run Gluten with gperftools",
"content": "For Spark on Yarn, we can change the submit script to run Gluten with gperftools. Add the following to the submit script: . export tcmalloc_path=/data2/zzb/gperftools-2.10/.libs/libtcmalloc_and_profiler.so # the path to the tcmalloc library export LD_PRELOAD=$tcmalloc_path,libch.so # load the library in the driver --files $tcmalloc_path # upload the library to the cluster --conf spark.executorEnv.LD_PRELOAD=./libtcmalloc_and_profiler.so,libch.so # load the library in the [...]
"url": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html#run-gluten-with-gperftools",
"relUrl": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html#run-gluten-with-gperftools"
- },"582": {
+ },"584": {
"doc": "UsingGperftoolsInCH",
"title": "Analyze the result",
"content": "We can get the result in the path we set in the previous step. For example, we can get the result in /tmp/gluten_heap_perf. We can use the following website to analyze the result: https://gperftools.github.io/gperftools/heapprofile.html https://gperftools.github.io/gperftools/cpuprofile.html . ",
"url": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html#analyze-the-result",
"relUrl": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html#analyze-the-result"
- },"583": {
+ },"585": {
"doc": "UsingGperftoolsInCH",
"title": "UsingGperftoolsInCH",
"content": "We need using gpertools to find the memory or CPU issue. That’s what this document is about. ",
"url": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html",
"relUrl": "/docs/v1.4.0/developers/UsingGperftoolsInCH.html"
- },"584": {
+ },"586": {
"doc": "UsingJemallocWithCH",
"title": "Change code of jemalloc",
"content": "Use libunwind instead of libgcc to get the backtrace which may cause process hangs. @@ -177,7 +177,8 @@ target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1) # At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracking. # # ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1). -target_compile_definitions (_jemalloc PRIV [...]
"url": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#change-code-of-jemalloc",
"relUrl": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#change-code-of-jemalloc"
- },"585": {
+ },"587": {
"doc": "UsingJemallocWithCH",
"title": "Get jeprof",
"content": "Change to the directory where you want to install jemalloc, and run the following commands: . cd $Clickhouse_SOURCE_PATH/contrib/jemalloc && ./autogen.sh && ./configure.sh && make -j8 . Then we get jeprof in the directory $Clickhouse_SOURCE_PATH/contrib/jemalloc/bin/jeprof. ",
"url": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#get-jeprof",
"relUrl": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#get-jeprof"
- },"586": {
+ },"588": {
"doc": "UsingJemallocWithCH",
"title": "Compiler libch.so",
"content": "Ensure to enable jemalloc -DENABLE_JEMALLOC=ON in cpp-ch/CMakeLists.txt, and compile libch.so. ",
"url": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#compiler-libchso",
"relUrl": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#compiler-libchso"
- },"587": {
+ },"589": {
"doc": "UsingJemallocWithCH",
"title": "Run Gluten with jemalloc heap tools",
"content": "For Yarn or thrift server, you need add the following to the submit script: . export MALLOC_CONF=\"prof:true,lg_prof_interval:30\" # enable jemalloc heap profiling . You can find more options in https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling. ",
"url": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#run-gluten-with-jemalloc-heap-tools",
"relUrl": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#run-gluten-with-jemalloc-heap-tools"
- },"588": {
+ },"590": {
"doc": "UsingJemallocWithCH",
"title": "Analyze the result",
"content": "After you get the heap file, you can use the following command to analyze the result: Check memory diff, so you can find which part of the code consume the memory. jeprof --svg /usr/lib/jvm/java-8-openjdk-amd64/bin/java --base=jeprof.3717358.1.i1.heap jeprof.3717358.30.i30.heap > diff.svg . You can find more usage about jeprof with jeprof --help. ",
"url": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#analyze-the-result",
"relUrl": "/docs/v1.4.0/developers/UsingJemallocWithCH.html#analyze-the-result"
- },"589": {
+ },"591": {
"doc": "UsingJemallocWithCH",
"title": "UsingJemallocWithCH",
"content": "We need using jemalloc to find the memory issue. That’s what this document is about. ",
"url": "/docs/v1.4.0/developers/UsingJemallocWithCH.html",
"relUrl": "/docs/v1.4.0/developers/UsingJemallocWithCH.html"
- },"590": {
+ },"592": {
"doc": "VeloxUDF",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": " ",
"url": "/docs/v1.4.0/developers/VeloxUDF.html#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"591": {
+ },"593": {
"doc": "VeloxUDF",
"title": "Introduction",
"content": "Velox backend supports User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF). Users can implement custom functions using the UDF interface provided by Velox and compile them into libraries. At runtime, these UDFs are registered alongside their Java implementations via CREATE TEMPORARY FUNCTION. Once registered, Gluten can parse and offload these UDFs to Velox during execution, meanwhile ensuring proper fallback to Java UDFs when necessary. ",
"url": "/docs/v1.4.0/developers/VeloxUDF.html#introduction",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html#introduction"
- },"592": {
+ },"594": {
"doc": "VeloxUDF",
"title": "Create and Build UDF/UDAF library",
"content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
"url": "/docs/v1.4.0/developers/VeloxUDF.html#create-and-build-udfudaf-library",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html#create-and-build-udfudaf-library"
- },"593": {
+ },"595": {
"doc": "VeloxUDF",
"title": "Using UDF/UDAF in Gluten",
"content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the library paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
"url": "/docs/v1.4.0/developers/VeloxUDF.html#using-udfudaf-in-gluten",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html#using-udfudaf-in-gluten"
- },"594": {
+ },"596": {
"doc": "VeloxUDF",
"title": "Try the example",
"content": "We provided Velox UDF examples in file MyUDF.cc and UDAF examples in file MyUDAF.cc. You need to build the gluten project with --build_example=ON to get the example libraries./dev/buildbundle-veloxbe.sh --build_examples=ON . Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ . Start spark-shell or spark-sql with below configuration . # Use the `--files` option to upload a library and configure its relative path --files /path/to/glute [...]
"url": "/docs/v1.4.0/developers/VeloxUDF.html#try-the-example",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html#try-the-example"
- },"595": {
+ },"597": {
"doc": "VeloxUDF",
"title": "Configurations",
"content": "| Parameters | Description | . | spark.gluten.sql.columnar.backend.velox.udfLibraryPaths | Path to the udf/udaf libraries. | . | spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths | Path to the udf/udaf libraries on driver node. Only applicable on yarn-client mode. | . | spark.gluten.sql.columnar.backend.velox.udfAllowTypeConversion | Whether to inject possible cast to convert mismatched data types from input to one registered signatures. | . ",
"url": "/docs/v1.4.0/developers/VeloxUDF.html#configurations",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html#configurations"
- },"596": {
+ },"598": {
"doc": "VeloxUDF",
"title": "Pandas UDFs (a.k.a. Vectorized UDFs)",
"content": " ",
"url": "/docs/v1.4.0/developers/VeloxUDF.html#pandas-udfs-aka-vectorized-udfs",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html#pandas-udfs-aka-vectorized-udfs"
- },"597": {
+ },"599": {
"doc": "VeloxUDF",
"title": "Introduction",
"content": "Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer doc. ",
"url": "/docs/v1.4.0/developers/VeloxUDF.html#introduction-1",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html#introduction-1"
- },"598": {
+ },"600": {
"doc": "VeloxUDF",
"title": "Using Pandas UDFs in Gluten with Velox Backend",
"content": "Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: . pip3 install pyspark==$SPARK_VERSION cython pip3 install pandas pyarrow . Gluten provides a config to control enable ColumnarArrowEvalPython or not, with true as default. spark.gluten.sql.columnar.arrowUdf . Then take following PySpark code for example: . from pyspark.sql.functions import pandas_udf, PandasUDFType import pyspark.sql.functions a [...]
"url": "/docs/v1.4.0/developers/VeloxUDF.html#using-pandas-udfs-in-gluten-with-velox-backend",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html#using-pandas-udfs-in-gluten-with-velox-backend"
- },"599": {
+ },"601": {
"doc": "VeloxUDF",
"title": "VeloxUDF",
"content": " ",
"url": "/docs/v1.4.0/developers/VeloxUDF.html",
"relUrl": "/docs/v1.4.0/developers/VeloxUDF.html"
- },"600": {
+ },"602": {
"doc": "clickhouse backend debug",
"title": "Debug CH Backend",
"content": " ",
"url": "/docs/v1.4.0/developers/clickhouse-backend-debug.html#debug-ch-backend",
"relUrl": "/docs/v1.4.0/developers/clickhouse-backend-debug.html#debug-ch-backend"
- },"601": {
+ },"603": {
"doc": "clickhouse backend debug",
"title": "Debug Java/Scala Code with IntelliJ IDEA",
"content": ". | Build Gluten ClickHouse Native Lib. export CMAKE_BUILD_TYPE=Release && bash ep/build-clickhouse/src/build_clickhouse.sh . libch.so will be generated in cpp-ch/build/utils/extern-local-engine/libch.so. | Maven Build Gluten ClickHouse with Profile mvn clean install -DskipTests -P delta -Pbackends-clickhouse -Pspark-3.3 -Pspark-ut . | Set Maven Profiles in IntelliJ IDEA . | Set Debug Configuration in IntelliJ IDEA . For example, debug GlutenMathExpressionsSuite. [...]
"url": "/docs/v1.4.0/developers/clickhouse-backend-debug.html#debug-javascala-code-with-intellij-idea",
"relUrl": "/docs/v1.4.0/developers/clickhouse-backend-debug.html#debug-javascala-code-with-intellij-idea"
- },"602": {
+ },"604": {
"doc": "clickhouse backend debug",
"title": "Debug Native Code with CLion",
"content": ". | Toolchains Settings . Some Clion versions have not supported lldb-18, you can manually set refer to CLion Custom LLDB. Or you can use gdb as default. | CMake Debug Configuration . CMake Options: -DENABLE_PROTOBUF=ON -DENABLE_TESTS=OFF -DENABLE_BENCHMARKS=ON -DENABLE_JEMALLOC=ON -DENABLE_MULTITARGET_CODE=ON -DENABLE_EXTERN_LOCAL_ENGINE=ON -DENABLE_ODBC=OFF -DENABLE_CAPNP=OFF -DENABLE_ROCKSDB=OFF -DENABLE_GRPC=OFF -DENABLE_RUST=OFF -DENABLE_H3=OFF -DENABLE_AMQPCPP=OFF - [...]
"url": "/docs/v1.4.0/developers/clickhouse-backend-debug.html#debug-native-code-with-clion",
"relUrl": "/docs/v1.4.0/developers/clickhouse-backend-debug.html#debug-native-code-with-clion"
- },"603": {
+ },"605": {
"doc": "clickhouse backend debug",
"title": "clickhouse backend debug",
"content": " ",
"url": "/docs/v1.4.0/developers/clickhouse-backend-debug.html",
"relUrl": "/docs/v1.4.0/developers/clickhouse-backend-debug.html"
- },"604": {
+ },"606": {
"doc": "improvement proposals",
"title": "Gluten Project Improvement Proposals (GPIP)",
"content": "The Gluten Project Improvement Proposals doc references the Spark SPIP documentation. The purpose of a GPIP is to inform and involve the user community in major improvements to the Gluten codebase throughout the development process to increase the likelihood that user needs are met. GPIPs should be used for significant user-facing or cutting-edge changes, not small incremental improvements. If your proposal meets the definition of GPIP, we recommend you to create a GPIP, [...]
"url": "/docs/v1.4.0/developers/improvement-proposals.html#gluten-project-improvement-proposals-gpip",
"relUrl": "/docs/v1.4.0/developers/improvement-proposals.html#gluten-project-improvement-proposals-gpip"
- },"605": {
+ },"607": {
"doc": "improvement proposals",
"title": "What is a GPIP?",
"content": "A GPIP is similar to a product requirement document commonly used in product management. A GPIP: . | Is a ticket labeled “GPIP” proposing a major improvement or change to Gluten | Follows the template defined below | Includes discussions on the ticket and dev@ list about the proposal | . ",
"url": "/docs/v1.4.0/developers/improvement-proposals.html#what-is-a-gpip",
"relUrl": "/docs/v1.4.0/developers/improvement-proposals.html#what-is-a-gpip"
- },"606": {
+ },"608": {
"doc": "improvement proposals",
"title": "Who?",
"content": "Any community member can help by discussing whether a GPIP is likely to meet their needs and propose GPIPs. Contributors can help by discussing whether a GPIP is likely to be technically feasible. Committers can help by discussing whether a GPIP aligns with long-term project goals, and by shepherding GPIPs. GPIP Author is any community member who authors a GPIP and is committed to pushing the change through the entire process. GPIP authorship can be transferred. GPIP Shep [...]
"url": "/docs/v1.4.0/developers/improvement-proposals.html#who",
"relUrl": "/docs/v1.4.0/developers/improvement-proposals.html#who"
- },"607": {
+ },"609": {
"doc": "improvement proposals",
"title": "GPIP Process",
"content": "Proposing a GPIP . Anyone may propose a GPIP, using the document template below. Please only submit a GPIP if you are willing to help, at least with discussion. After a GPIP is created, the author should email dev@gluten.apache.org to notify the community of the GPIP, and discussions should ensue on the ticket. If a GPIP is too small or incremental and should have been done through the normal ticket process, a committer should remove the GPIP label. GPIP Document Template [...]
"url": "/docs/v1.4.0/developers/improvement-proposals.html#gpip-process",
"relUrl": "/docs/v1.4.0/developers/improvement-proposals.html#gpip-process"
- },"608": {
+ },"610": {
"doc": "improvement proposals",
"title": "improvement proposals",
"content": " ",
"url": "/docs/v1.4.0/developers/improvement-proposals.html",
"relUrl": "/docs/v1.4.0/developers/improvement-proposals.html"
- },"609": {
+ },"611": {
"doc": "Developers",
"title": "Gluten Developers",
"content": "This document provides a developer overview of the project and covers the following topics: . ",
"url": "/archives/v1.4.0/developers#gluten-developers",
"relUrl": "/archives/v1.4.0/developers#gluten-developers"
- },"610": {
+ },"612": {
"doc": "Developers",
"title": "Developers",
"content": " ",
"url": "/archives/v1.4.0/developers",
"relUrl": "/archives/v1.4.0/developers"
- },"611": {
+ },"613": {
"doc": "velox backend CI",
"title": "Velox Backend CI",
"content": "GHA workflows are defined under .github/workflows/. ",
"url": "/docs/v1.4.0/developers/velox-backend-CI.html#velox-backend-ci",
"relUrl": "/docs/v1.4.0/developers/velox-backend-CI.html#velox-backend-ci"
- },"612": {
+ },"614": {
"doc": "velox backend CI",
"title": "Docker Build",
"content": "We have a weekly job defined in docker_image.yml to build docker images based on Dockerfile.centos7-static-build and Dockerfile.centos8-dynamic-build for CI verification. ",
"url": "/docs/v1.4.0/developers/velox-backend-CI.html#docker-build",
"relUrl": "/docs/v1.4.0/developers/velox-backend-CI.html#docker-build"
- },"613": {
+ },"615": {
"doc": "velox backend CI",
"title": "Vcpkg Caching",
"content": "Gluten main branch is pulled down during docker build. And vcpkg will cache binary data of all dependencies defined under dev/vcpkg. These binary data is cached into /var/cache/vcpkg and CI job can re-use them in new build. By setting VCPKG_BINARY_SOURCES=clear in env., reusing vcpkg cache can be disabled. ",
"url": "/docs/v1.4.0/developers/velox-backend-CI.html#vcpkg-caching",
"relUrl": "/docs/v1.4.0/developers/velox-backend-CI.html#vcpkg-caching"
- },"614": {
+ },"616": {
"doc": "velox backend CI",
"title": "Arrow Libs Pre-installation",
"content": "Arrow libs are pre-installed in docker, assuming they are not actively changed, then not necessarily to be re-built every time. ",
"url": "/docs/v1.4.0/developers/velox-backend-CI.html#arrow-libs-pre-installation",
"relUrl": "/docs/v1.4.0/developers/velox-backend-CI.html#arrow-libs-pre-installation"
- },"615": {
+ },"617": {
"doc": "velox backend CI",
"title": "Updating Docker Image",
"content": "Two GitHub secrets DOCKERHUB_USER & DOCKERHUB_TOKEN can be used to push docker image to docker hub: https://hub.docker.com/r/apache/gluten/tags. Note GitHub secrets are not retrievable in PR from forked repo. ",
"url": "/docs/v1.4.0/developers/velox-backend-CI.html#updating-docker-image",
"relUrl": "/docs/v1.4.0/developers/velox-backend-CI.html#updating-docker-image"
- },"616": {
+ },"618": {
"doc": "velox backend CI",
"title": "velox backend CI",
"content": " ",
"url": "/docs/v1.4.0/developers/velox-backend-CI.html",
"relUrl": "/docs/v1.4.0/developers/velox-backend-CI.html"
- },"617": {
+ },"619": {
"doc": "velox backend build in docker",
"title": "Static link",
"content": "The static link approach builds all dependency libraries in vcpkg for both Velox and Gluten. It then statically links these libraries into libvelox.so and libgluten.so, enabling the build of Gluten on any Linux OS on x86 platforms with 64G memory. However we only verified on Centos-7/8/9 and Ubuntu 20.04/22.04. Please submit an issue if it fails on your OS. Here is the dependency libraries required on target system, they are the essential libraries pre-installed in every [...]
"url": "/docs/v1.4.0/developers/velox-backend-build-in-docker.html#static-link",
"relUrl": "/docs/v1.4.0/developers/velox-backend-build-in-docker.html#static-link"
- },"618": {
+ },"620": {
"doc": "velox backend build in docker",
"title": "Dynamic link",
"content": "The dynamic link approach needs to install the dependencies libraries. It then dynamically link the .so files into libvelox.so and libgluten.so. Currently, Centos-7/8/9 and Ubuntu 20.04/22.04 are supported to build Gluten Velox backend dynamically. The ‘dockerfile’ to build Gluten jar: . FROM apache/gluten:centos-8-jdk8 # Build Gluten Jar RUN source /opt/rh/devtoolset-11/enable && \\ git clone https://github.com/apache/incubator-gluten.git && \\ cd incubat [...]
"url": "/docs/v1.4.0/developers/velox-backend-build-in-docker.html#dynamic-link",
"relUrl": "/docs/v1.4.0/developers/velox-backend-build-in-docker.html#dynamic-link"
- },"619": {
+ },"621": {
"doc": "velox backend build in docker",
"title": "velox backend build in docker",
"content": "Currently, we have two way to build Gluten, static link or dynamic link. ",
"url": "/docs/v1.4.0/developers/velox-backend-build-in-docker.html",
"relUrl": "/docs/v1.4.0/developers/velox-backend-build-in-docker.html"
- },"620": {
+ },"622": {
"doc": "velox function development guide",
"title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
"content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
"url": "/docs/v1.4.0/developers/velox-function-development-guide.html#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
"relUrl": "/docs/v1.4.0/developers/velox-function-development-guide.html#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"621": {
+ },"623": {
"doc": "velox function development guide",
"title": "velox function development guide",
"content": " ",
"url": "/docs/v1.4.0/developers/velox-function-development-guide.html",
"relUrl": "/docs/v1.4.0/developers/velox-function-development-guide.html"
- },"622": {
+ },"624": {
"doc": "ClickHouse",
"title": "ClickHouse Backend",
"content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
"url": "/docs/v1.4.0/get-started/ClickHouse.html#clickhouse-backend",
"relUrl": "/docs/v1.4.0/get-started/ClickHouse.html#clickhouse-backend"
- },"623": {
+ },"625": {
"doc": "ClickHouse",
"title": "ClickHouse",
"content": " ",
"url": "/docs/v1.4.0/get-started/ClickHouse.html",
"relUrl": "/docs/v1.4.0/get-started/ClickHouse.html"
- },"624": {
+ },"626": {
"doc": "GlutenUI",
"title": "Gluten UI",
"content": " ",
"url": "/docs/v1.4.0/get-started/GlutenUI.html#gluten-ui",
"relUrl": "/docs/v1.4.0/get-started/GlutenUI.html#gluten-ui"
- },"625": {
+ },"627": {
"doc": "GlutenUI",
"title": "Gluten event",
"content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
"url": "/docs/v1.4.0/get-started/GlutenUI.html#gluten-event",
"relUrl": "/docs/v1.4.0/get-started/GlutenUI.html#gluten-event"
- },"626": {
+ },"628": {
"doc": "GlutenUI",
"title": "SQL tab",
"content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
"url": "/docs/v1.4.0/get-started/GlutenUI.html#sql-tab",
"relUrl": "/docs/v1.4.0/get-started/GlutenUI.html#sql-tab"
- },"627": {
+ },"629": {
"doc": "GlutenUI",
"title": "History server",
"content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
"url": "/docs/v1.4.0/get-started/GlutenUI.html#history-server",
"relUrl": "/docs/v1.4.0/get-started/GlutenUI.html#history-server"
- },"628": {
+ },"630": {
"doc": "GlutenUI",
"title": "GlutenUI",
"content": " ",
"url": "/docs/v1.4.0/get-started/GlutenUI.html",
"relUrl": "/docs/v1.4.0/get-started/GlutenUI.html"
- },"629": {
+ },"631": {
"doc": "Velox",
"title": "Supported Version",
"content": "| Type | Version | . | Spark | 3.2.2, 3.3.1, 3.4.4, 3.5.2 | . | OS | Ubuntu20.04/22.04, Centos7/8 | . | jdk | openjdk8/jdk17 | . | scala | 2.12 | . ",
"url": "/docs/v1.4.0/get-started/Velox.html#supported-version",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#supported-version"
- },"630": {
+ },"632": {
"doc": "Velox",
"title": "Prerequisite",
"content": "Currently, with static build Gluten+Velox backend supports all the Linux OSes, but is only tested on Ubuntu20.04/Ubuntu22.04/Centos7/Centos8. With dynamic build, Gluten+Velox backend support Ubuntu20.04/Ubuntu22.04/Centos7/Centos8 and their variants. Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.4 and 3.5.2. We need to set up the JAVA_HOME env. Currently, Gluten supports java 8 and java 17. For x86_64 . ## make sure jdk8 is used export JAVA_HOME [...]
"url": "/docs/v1.4.0/get-started/Velox.html#prerequisite",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#prerequisite"
- },"631": {
+ },"633": {
"doc": "Velox",
"title": "Build Gluten with Velox Backend",
"content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . First time build for all supported spark versions./dev/buildbundle-veloxbe.sh . After a complete build, if only some gluten code is changed, you can use the following command to skip building arrow and setting up build dependencies./dev/buildbundle-veloxbe.sh --build_arrow=OFF [...]
"url": "/docs/v1.4.0/get-started/Velox.html#build-gluten-with-velox-backend",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#build-gluten-with-velox-backend"
- },"632": {
+ },"634": {
"doc": "Velox",
"title": "Dependency library deployment",
"content": "With build option enable_vcpkg=ON, all dependency libraries will be statically linked to libvelox.so and libgluten.so which are packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed ( build in docker ). On host with some libraries like jemalloc ins [...]
"url": "/docs/v1.4.0/get-started/Velox.html#dependency-library-deployment",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#dependency-library-deployment"
- },"633": {
+ },"635": {
"doc": "Velox",
"title": "Remote storage support",
"content": " ",
"url": "/docs/v1.4.0/get-started/Velox.html#remote-storage-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#remote-storage-support"
- },"634": {
+ },"636": {
"doc": "Velox",
"title": "HDFS support",
"content": "Gluten supports dynamically loading both libhdfs.so and libhdfs3.so at runtime by using dlopen, allowing the JVM to load the appropriate shared library file as needed. This means you do not need to set the library path during the compilation phase. To enable this functionality, you must set the JAVA_HOME and HADOOP_HOME environment variables. Gluten will then locate and load the ${HADOOP_HOME}/lib/native/libhdfs.so file at runtime. If you prefer to use libhdfs3.so instead [...]
"url": "/docs/v1.4.0/get-started/Velox.html#hdfs-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#hdfs-support"
- },"635": {
+ },"637": {
"doc": "Velox",
"title": "Azure Blob File System (ABFS) support",
"content": "Velox supports ABFS with the open source Azure SDK for C++ and Gluten uses the Velox ABFS connector to connect with ABFS. The build option for ABFS (enable_abfs) must be set to enable this feature as listed below. cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_abfs=ON . Please refer Velox ABFS part for more detailed configurations. ",
"url": "/docs/v1.4.0/get-started/Velox.html#azure-blob-file-system-abfs-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#azure-blob-file-system-abfs-support"
- },"636": {
+ },"638": {
"doc": "Velox",
"title": "AWS S3 support",
"content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to access S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
"url": "/docs/v1.4.0/get-started/Velox.html#aws-s3-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#aws-s3-support"
- },"637": {
+ },"639": {
"doc": "Velox",
"title": "GCS support",
"content": "Please refer GCS . ",
"url": "/docs/v1.4.0/get-started/Velox.html#gcs-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#gcs-support"
- },"638": {
+ },"640": {
"doc": "Velox",
"title": "Remote Shuffle Service Support",
"content": " ",
"url": "/docs/v1.4.0/get-started/Velox.html#remote-shuffle-service-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#remote-shuffle-service-support"
- },"639": {
+ },"641": {
"doc": "Velox",
"title": "Celeborn support",
"content": "Gluten with velox backend supports Celeborn as remote shuffle service. Currently, the supported Celeborn versions are 0.3.x, 0.4.x and 0.5.x. Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable celeborn profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests . Then add the Gluten and Spa [...]
"url": "/docs/v1.4.0/get-started/Velox.html#celeborn-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#celeborn-support"
- },"640": {
+ },"642": {
"doc": "Velox",
"title": "Uniffle support",
"content": "Uniffle with velox backend supports Uniffle as remote shuffle service. Currently, the supported Uniffle versions are 0.9.2. First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with uniffle. When compiling the Gluten Java module, it’s required to enable uniffle profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests . Then add the Uniffle and Spark Celeborn Client packages to your Spark application’s classpath(usu [...]
"url": "/docs/v1.4.0/get-started/Velox.html#uniffle-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#uniffle-support"
- },"641": {
+ },"643": {
"doc": "Velox",
"title": "Datalake Framework Support",
"content": " ",
"url": "/docs/v1.4.0/get-started/Velox.html#datalake-framework-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#datalake-framework-support"
- },"642": {
+ },"644": {
"doc": "Velox",
"title": "DeltaLake Support",
"content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Once built successfully, delta features will be included in gluten-velox-bundle-X jar. Then you can query delta table by gluten/velox without scan’s fallback. Gluten with velox backends also support the column mapping of delta tables. About column mapping, see more [...]
"url": "/docs/v1.4.0/get-started/Velox.html#deltalake-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#deltalake-support"
- },"643": {
+ },"645": {
"doc": "Velox",
"title": "Iceberg Support",
"content": "Gluten with velox backend supports Iceberg table. Currently, both reading COW (Copy-On-Write) and MOR (Merge-On-Read) tables are supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan’s fallback. ",
"url": "/docs/v1.4.0/get-started/Velox.html#iceberg-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#iceberg-support"
- },"644": {
+ },"646": {
"doc": "Velox",
"title": "Hudi Support",
"content": "Gluten with velox backend supports Hudi table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-hudi module by a hudi profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Phudi -DskipTests . Once built successfully, hudi features will be included in gluten-velox-bundle-X jar. Then you can query hudi COW table by gluten/velox without scan’s fallback. ",
"url": "/docs/v1.4.0/get-started/Velox.html#hudi-support",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#hudi-support"
- },"645": {
+ },"647": {
"doc": "Velox",
"title": "Coverage",
"content": "Spark3.3 has 387 functions in total. ~240 are commonly used. To get the support status of all Spark built-in functions, please refer to Velox Backend’s Supported Operators & Functions. Velox doesn’t support ANSI mode), so as Gluten. Once ANSI mode is enabled in Spark config, Gluten will fallback to Vanilla Spark. To identify what can be offloaded in a query and detailed fallback reasons, user can follow below steps to retrieve corresponding logs. 1) Enable Gluten by p [...]
"url": "/docs/v1.4.0/get-started/Velox.html#coverage",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#coverage"
- },"646": {
+ },"648": {
"doc": "Velox",
"title": "Spill",
"content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
"url": "/docs/v1.4.0/get-started/Velox.html#spill",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#spill"
- },"647": {
+ },"649": {
"doc": "Velox",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": "Please check the VeloxNativeUDF.md for more detailed usage and configurations. ",
"url": "/docs/v1.4.0/get-started/Velox.html#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"648": {
+ },"650": {
"doc": "Velox",
"title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
"content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. You may refer to the notebook we used to do the performance test. ",
"url": "/docs/v1.4.0/get-started/Velox.html#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
- },"649": {
+ },"651": {
"doc": "Velox",
"title": "Data preparation",
"content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. ",
"url": "/docs/v1.4.0/get-started/Velox.html#data-preparation",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#data-preparation"
- },"650": {
+ },"652": {
"doc": "Velox",
"title": "Submit the Spark SQL job",
"content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
"url": "/docs/v1.4.0/get-started/Velox.html#submit-the-spark-sql-job",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#submit-the-spark-sql-job"
- },"651": {
+ },"653": {
"doc": "Velox",
"title": "Result",
"content": "wholestagetransformer indicates that the offloading works. ",
"url": "/docs/v1.4.0/get-started/Velox.html#result",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#result"
- },"652": {
+ },"654": {
"doc": "Velox",
"title": "Performance",
"content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
"url": "/docs/v1.4.0/get-started/Velox.html#performance",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#performance"
- },"653": {
+ },"655": {
"doc": "Velox",
"title": "Gluten UI",
"content": "Please refer Gluten UI . ",
"url": "/docs/v1.4.0/get-started/Velox.html#gluten-ui",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#gluten-ui"
- },"654": {
+ },"656": {
"doc": "Velox",
"title": "Gluten Native Plan Summary",
"content": " ",
"url": "/docs/v1.4.0/get-started/Velox.html#gluten-native-plan-summary",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#gluten-native-plan-summary"
- },"655": {
+ },"657": {
"doc": "Velox",
"title": "Gluten Implicits",
"content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
"url": "/docs/v1.4.0/get-started/Velox.html#gluten-implicits",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#gluten-implicits"
- },"656": {
+ },"658": {
"doc": "Velox",
"title": "Native Plan in Spark’s Explain Output",
"content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten shows the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \ [...]
"url": "/docs/v1.4.0/get-started/Velox.html#native-plan-in-sparks-explain-output",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#native-plan-in-sparks-explain-output"
- },"657": {
+ },"659": {
"doc": "Velox",
"title": "Native Plan with Stats",
"content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
"url": "/docs/v1.4.0/get-started/Velox.html#native-plan-with-stats",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#native-plan-with-stats"
- },"658": {
+ },"660": {
"doc": "Velox",
"title": "Using Stage-Level Resource Adjustment to Avoid OOM(Experimental)",
"content": "see more here . ",
"url": "/docs/v1.4.0/get-started/Velox.html#using-stage-level-resource-adjustment-to-avoid-oomexperimental",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#using-stage-level-resource-adjustment-to-avoid-oomexperimental"
- },"659": {
+ },"661": {
"doc": "Velox",
"title": "Broadcast Build Relations to Off-Heap(Experimental)",
"content": "The experimental feature Off-Heap Broadcast Build Relations aims to mitigate out-of-memory (OOM) issues caused by heap memory consumption during broadcast operations. Detailed design can be found here . Purpose & how it works . | Avoid OOM: Prevent OOM errors when broadcasting large datasets. | Reduce Heap Memory Usage: Store broadcast build relations in Spark off-heap memory instead of on-heap memory | . Configuration . Enable Off-Heap Broadcast . To enable this feat [...]
"url": "/docs/v1.4.0/get-started/Velox.html#broadcast-build-relations-to-off-heapexperimental",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#broadcast-build-relations-to-off-heapexperimental"
- },"660": {
+ },"662": {
"doc": "Velox",
"title": "Accelerators",
"content": "Please refer HBM QAT IAA for details . ",
"url": "/docs/v1.4.0/get-started/Velox.html#accelerators",
"relUrl": "/docs/v1.4.0/get-started/Velox.html#accelerators"
- },"661": {
+ },"663": {
"doc": "Velox",
"title": "Velox",
"content": " ",
"url": "/docs/v1.4.0/get-started/Velox.html",
"relUrl": "/docs/v1.4.0/get-started/Velox.html"
- },"662": {
+ },"664": {
"doc": "VeloxABFS",
"title": "Working with ABFS",
"content": " ",
"url": "/docs/v1.4.0/get-started/VeloxABFS.html#working-with-abfs",
"relUrl": "/docs/v1.4.0/get-started/VeloxABFS.html#working-with-abfs"
- },"663": {
+ },"665": {
"doc": "VeloxABFS",
"title": "Configuring ABFS Access Token",
"content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
"url": "/docs/v1.4.0/get-started/VeloxABFS.html#configuring-abfs-access-token",
"relUrl": "/docs/v1.4.0/get-started/VeloxABFS.html#configuring-abfs-access-token"
- },"664": {
+ },"666": {
"doc": "VeloxABFS",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/docs/v1.4.0/get-started/VeloxABFS.html#local-caching-support",
"relUrl": "/docs/v1.4.0/get-started/VeloxABFS.html#local-caching-support"
- },"665": {
+ },"667": {
"doc": "VeloxABFS",
"title": "VeloxABFS",
"content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
"url": "/docs/v1.4.0/get-started/VeloxABFS.html",
"relUrl": "/docs/v1.4.0/get-started/VeloxABFS.html"
- },"666": {
+ },"668": {
"doc": "VeloxGCS",
"title": "Working with GCS",
"content": " ",
"url": "/docs/v1.4.0/get-started/VeloxGCS.html#working-with-gcs",
"relUrl": "/docs/v1.4.0/get-started/VeloxGCS.html#working-with-gcs"
- },"667": {
+ },"669": {
"doc": "VeloxGCS",
"title": "Installing the gcloud CLI",
"content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
"url": "/docs/v1.4.0/get-started/VeloxGCS.html#installing-the-gcloud-cli",
"relUrl": "/docs/v1.4.0/get-started/VeloxGCS.html#installing-the-gcloud-cli"
- },"668": {
+ },"670": {
"doc": "VeloxGCS",
"title": "Configuring GCS using a user account",
"content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
"url": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-using-a-user-account",
"relUrl": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-using-a-user-account"
- },"669": {
+ },"671": {
"doc": "VeloxGCS",
"title": "Configuring GCS using a credential file",
"content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credentials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to t [...]
"url": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-using-a-credential-file",
"relUrl": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-using-a-credential-file"
- },"670": {
+ },"672": {
"doc": "VeloxGCS",
"title": "Configuring GCS endpoints",
"content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
"url": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-endpoints",
"relUrl": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-endpoints"
- },"671": {
+ },"673": {
"doc": "VeloxGCS",
"title": "Configuring GCS max retry count",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
"url": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-max-retry-count",
"relUrl": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-max-retry-count"
- },"672": {
+ },"674": {
"doc": "VeloxGCS",
"title": "Configuring GCS max retry time",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retrying (10s, 1m, etc). ",
"url": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-max-retry-time",
"relUrl": "/docs/v1.4.0/get-started/VeloxGCS.html#configuring-gcs-max-retry-time"
- },"673": {
+ },"675": {
"doc": "VeloxGCS",
"title": "VeloxGCS",
"content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/docs/v1.4.0/get-started/VeloxGCS.html",
"relUrl": "/docs/v1.4.0/get-started/VeloxGCS.html"
- },"674": {
+ },"676": {
"doc": "VeloxHBM",
"title": "High-Bandwidth Memory (HBM) support",
"content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
"url": "/docs/v1.4.0/get-started/VeloxHBM.html#high-bandwidth-memory-hbm-support",
"relUrl": "/docs/v1.4.0/get-started/VeloxHBM.html#high-bandwidth-memory-hbm-support"
- },"675": {
+ },"677": {
"doc": "VeloxHBM",
"title": "Build Gluten with HBM",
"content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.3 and 3.5.1./dev/buildbundle-veloxbe.sh -- [...]
"url": "/docs/v1.4.0/get-started/VeloxHBM.html#build-gluten-with-hbm",
"relUrl": "/docs/v1.4.0/get-started/VeloxHBM.html#build-gluten-with-hbm"
- },"676": {
+ },"678": {
"doc": "VeloxHBM",
"title": "Configure and enable HBM in Spark Application",
"content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
"url": "/docs/v1.4.0/get-started/VeloxHBM.html#configure-and-enable-hbm-in-spark-application",
"relUrl": "/docs/v1.4.0/get-started/VeloxHBM.html#configure-and-enable-hbm-in-spark-application"
- },"677": {
+ },"679": {
"doc": "VeloxHBM",
"title": "VeloxHBM",
"content": " ",
"url": "/docs/v1.4.0/get-started/VeloxHBM.html",
"relUrl": "/docs/v1.4.0/get-started/VeloxHBM.html"
- },"678": {
+ },"680": {
"doc": "VeloxIAA",
"title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
"content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
"url": "/docs/v1.4.0/get-started/VeloxIAA.html#intel-in-memory-analytics-accelerator-iaaiax-support",
"relUrl": "/docs/v1.4.0/get-started/VeloxIAA.html#intel-in-memory-analytics-accelerator-iaaiax-support"
- },"679": {
+ },"681": {
"doc": "VeloxIAA",
"title": "Build Gluten with IAA",
"content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
"url": "/docs/v1.4.0/get-started/VeloxIAA.html#build-gluten-with-iaa",
"relUrl": "/docs/v1.4.0/get-started/VeloxIAA.html#build-gluten-with-iaa"
- },"680": {
+ },"682": {
"doc": "VeloxIAA",
"title": "Enable IAA with Gzip Compression for shuffle compression",
"content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
"url": "/docs/v1.4.0/get-started/VeloxIAA.html#enable-iaa-with-gzip-compression-for-shuffle-compression",
"relUrl": "/docs/v1.4.0/get-started/VeloxIAA.html#enable-iaa-with-gzip-compression-for-shuffle-compression"
- },"681": {
+ },"683": {
"doc": "VeloxIAA",
"title": "IAA references",
"content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
"url": "/docs/v1.4.0/get-started/VeloxIAA.html#iaa-references",
"relUrl": "/docs/v1.4.0/get-started/VeloxIAA.html#iaa-references"
- },"682": {
+ },"684": {
"doc": "VeloxIAA",
"title": "VeloxIAA",
"content": " ",
"url": "/docs/v1.4.0/get-started/VeloxIAA.html",
"relUrl": "/docs/v1.4.0/get-started/VeloxIAA.html"
- },"683": {
+ },"685": {
"doc": "VeloxIceberg",
"title": "Iceberg Support in Velox Backend",
"content": " ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#iceberg-support-in-velox-backend",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#iceberg-support-in-velox-backend"
- },"684": {
+ },"686": {
"doc": "VeloxIceberg",
"title": "Supported Spark version",
"content": "All the spark version is supported, but for convenience, only Spark 3.4 is well tested. Now only read is supported in Gluten. ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#supported-spark-version",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#supported-spark-version"
- },"685": {
+ },"687": {
"doc": "VeloxIceberg",
"title": "Support Status",
"content": "Following value indicates the iceberg support progress: . | Value | Description | . | Offload | Offload to the Velox backend | . | PartialOffload | Some operators offload and some fallback | . | Fallback | Fallback to spark to execute | . | Exception | Cannot fallback by some conditions, throw the exception | . | ResultMismatch | Some hidden bug may cause result mismatch, especially for some corner case | . ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#support-status",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#support-status"
- },"686": {
+ },"688": {
"doc": "VeloxIceberg",
"title": "Adding catalogs",
"content": "Fallback . ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#adding-catalogs",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#adding-catalogs"
- },"687": {
+ },"689": {
"doc": "VeloxIceberg",
"title": "Creating a table",
"content": "Fallback . ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#creating-a-table",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#creating-a-table"
- },"688": {
+ },"690": {
"doc": "VeloxIceberg",
"title": "Writing",
"content": "Fallback . INSERT INTO local.db.table VALUES (1, 'a'), (2, 'b'), (3, 'c'); . PartialOffload . The write is fallback while read is offload. INSERT INTO local.db.table SELECT id, data FROM source WHERE length(data) = 1; . ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#writing",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#writing"
- },"689": {
+ },"691": {
"doc": "VeloxIceberg",
"title": "Reading",
"content": "Read data . Offload/Fallback . | Table Type | No Delete | Position Delete | Equality Delete | . | unpartition | Offload | Offload | Fallback | . | partition | Fallback mostly | Fallback mostly | Fallback | . | metadata | Fallback | | | . Offload the simple query. SELECT count(1) as count, data FROM local.db.table GROUP BY data; . If delete by Spark and copy on read, will generate position delete file, the query may offload. If delete by Flink, may generate the equalit [...]
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#reading",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#reading"
- },"690": {
+ },"692": {
"doc": "VeloxIceberg",
"title": "DataType",
"content": "Timestamptz in orc format is not supported, throws exception. UUID type and Fixed type is fallback. ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#datatype",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#datatype"
- },"691": {
+ },"693": {
"doc": "VeloxIceberg",
"title": "Format",
"content": "PartialOffload . Supports parquet and orc format. Not support avro format. ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#format",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#format"
- },"692": {
+ },"694": {
"doc": "VeloxIceberg",
"title": "SQL",
"content": "Only support SELECT. ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#sql",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#sql"
- },"693": {
+ },"695": {
"doc": "VeloxIceberg",
"title": "Configuration",
"content": "Catalogs . Supports all the catalog options, which is not used in native engine. SQL Extensions . Fallback . Supports the option spark.sql.extensions, fallback the SQL command CALL. Runtime configuration . Read options . | Spark option | Status | . | snapshot-id | Support | . | as-of-timestamp | Support | . | split-size | Support | . | lookback | Support | . | file-open-cost | Support | . | vectorization-enabled | Not Support | . | batch-size | Not Support | . ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html#configuration",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html#configuration"
- },"694": {
+ },"696": {
"doc": "VeloxIceberg",
"title": "VeloxIceberg",
"content": " ",
"url": "/docs/v1.4.0/get-started/VeloxIceberg.html",
"relUrl": "/docs/v1.4.0/get-started/VeloxIceberg.html"
- },"695": {
+ },"697": {
"doc": "VeloxLocalCache",
"title": "VeloxLocalCache",
"content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
"url": "/docs/v1.4.0/get-started/VeloxLocalCache.html",
"relUrl": "/docs/v1.4.0/get-started/VeloxLocalCache.html"
- },"696": {
+ },"698": {
"doc": "VeloxQAT",
"title": "Intel® QuickAssist Technology (QAT) support",
"content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
"url": "/docs/v1.4.0/get-started/VeloxQAT.html#intel-quickassist-technology-qat-support",
"relUrl": "/docs/v1.4.0/get-started/VeloxQAT.html#intel-quickassist-technology-qat-support"
- },"697": {
+ },"699": {
"doc": "VeloxQAT",
"title": "Software Requirements",
"content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
"url": "/docs/v1.4.0/get-started/VeloxQAT.html#software-requirements",
"relUrl": "/docs/v1.4.0/get-started/VeloxQAT.html#software-requirements"
- },"698": {
+ },"700": {
"doc": "VeloxQAT",
"title": "Build Gluten with QAT",
"content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
"url": "/docs/v1.4.0/get-started/VeloxQAT.html#build-gluten-with-qat",
"relUrl": "/docs/v1.4.0/get-started/VeloxQAT.html#build-gluten-with-qat"
- },"699": {
+ },"701": {
"doc": "VeloxQAT",
"title": "Enable QAT with Gzip/Zstd for shuffle compression",
"content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
"url": "/docs/v1.4.0/get-started/VeloxQAT.html#enable-qat-with-gzipzstd-for-shuffle-compression",
"relUrl": "/docs/v1.4.0/get-started/VeloxQAT.html#enable-qat-with-gzipzstd-for-shuffle-compression"
- },"700": {
+ },"702": {
"doc": "VeloxQAT",
"title": "QAT driver references",
"content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
"url": "/docs/v1.4.0/get-started/VeloxQAT.html#qat-driver-references",
"relUrl": "/docs/v1.4.0/get-started/VeloxQAT.html#qat-driver-references"
- },"701": {
+ },"703": {
"doc": "VeloxQAT",
"title": "VeloxQAT",
"content": " ",
"url": "/docs/v1.4.0/get-started/VeloxQAT.html",
"relUrl": "/docs/v1.4.0/get-started/VeloxQAT.html"
- },"702": {
+ },"704": {
"doc": "VeloxS3",
"title": "Working with S3",
"content": " ",
"url": "/docs/v1.4.0/get-started/VeloxS3.html#working-with-s3",
"relUrl": "/docs/v1.4.0/get-started/VeloxS3.html#working-with-s3"
- },"703": {
+ },"705": {
"doc": "VeloxS3",
"title": "Configuring S3 endpoint",
"content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
"url": "/docs/v1.4.0/get-started/VeloxS3.html#configuring-s3-endpoint",
"relUrl": "/docs/v1.4.0/get-started/VeloxS3.html#configuring-s3-endpoint"
- },"704": {
+ },"706": {
"doc": "VeloxS3",
"title": "Configuring S3 instance credentials",
"content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
"url": "/docs/v1.4.0/get-started/VeloxS3.html#configuring-s3-instance-credentials",
"relUrl": "/docs/v1.4.0/get-started/VeloxS3.html#configuring-s3-instance-credentials"
- },"705": {
+ },"707": {
"doc": "VeloxS3",
"title": "Configuring S3 IAM roles",
"content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
"url": "/docs/v1.4.0/get-started/VeloxS3.html#configuring-s3-iam-roles",
"relUrl": "/docs/v1.4.0/get-started/VeloxS3.html#configuring-s3-iam-roles"
- },"706": {
+ },"708": {
"doc": "VeloxS3",
"title": "Other authentatication methods are not supported yet",
"content": ". | AWS temporary credential | . ",
"url": "/docs/v1.4.0/get-started/VeloxS3.html#other-authentatication-methods-are-not-supported-yet",
"relUrl": "/docs/v1.4.0/get-started/VeloxS3.html#other-authentatication-methods-are-not-supported-yet"
- },"707": {
+ },"709": {
"doc": "VeloxS3",
"title": "Log granularity of AWS C++ SDK in velox",
"content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
"url": "/docs/v1.4.0/get-started/VeloxS3.html#log-granularity-of-aws-c-sdk-in-velox",
"relUrl": "/docs/v1.4.0/get-started/VeloxS3.html#log-granularity-of-aws-c-sdk-in-velox"
- },"708": {
+ },"710": {
"doc": "VeloxS3",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/docs/v1.4.0/get-started/VeloxS3.html#local-caching-support",
"relUrl": "/docs/v1.4.0/get-started/VeloxS3.html#local-caching-support"
- },"709": {
+ },"711": {
"doc": "VeloxS3",
"title": "VeloxS3",
"content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/docs/v1.4.0/get-started/VeloxS3.html",
"relUrl": "/docs/v1.4.0/get-started/VeloxS3.html"
- },"710": {
+ },"712": {
"doc": "VeloxStageResourceAdj",
"title": "Using Stage-Level Resource Adjustment to Avoid OOM(Experimental)",
"content": "Overview . One major advantage of Apache Gluten is its ability to significantly reduce memory requirements per executor—potentially by up to half—when entire stages are offloaded to the native engine. This engine primarily relies on off-heap memory with minimal on-heap usage. However, when stages contain fallback operators that utilize the JVM engine, the on-heap memory size must be increased, leading to even higher memory demands per executor. This challenge has posed si [...]
"url": "/docs/v1.4.0/get-started/VeloxStageResourceAdj.html#using-stage-level-resource-adjustment-to-avoid-oomexperimental",
"relUrl": "/docs/v1.4.0/get-started/VeloxStageResourceAdj.html#using-stage-level-resource-adjustment-to-avoid-oomexperimental"
- },"711": {
+ },"713": {
"doc": "VeloxStageResourceAdj",
"title": "VeloxStageResourceAdj",
"content": " ",
"url": "/docs/v1.4.0/get-started/VeloxStageResourceAdj.html",
"relUrl": "/docs/v1.4.0/get-started/VeloxStageResourceAdj.html"
- },"712": {
+ },"714": {
"doc": "build guide",
"title": "Build Parameters",
"content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | enable_jemalloc_stats | Print jemalloc stats for debugging. | [...]
"url": "/docs/v1.4.0/get-started/build-guide.html#build-parameters",
"relUrl": "/docs/v1.4.0/get-started/build-guide.html#build-parameters"
- },"713": {
+ },"715": {
"doc": "build guide",
"title": "Gluten Jar for Deployment",
"content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.4 | 3.4 | 2.12 | . | 3.5.2 | 3.5 | 2.12 | . ",
"url": "/docs/v1.4.0/get-started/build-guide.html#gluten-jar-for-deployment",
"relUrl": "/docs/v1.4.0/get-started/build-guide.html#gluten-jar-for-deployment"
- },"714": {
+ },"716": {
"doc": "build guide",
"title": "build guide",
"content": " ",
"url": "/docs/v1.4.0/get-started/build-guide.html",
"relUrl": "/docs/v1.4.0/get-started/build-guide.html"
- },"715": {
+ },"717": {
"doc": "Getting-Started",
"title": "Getting Started with Gluten for Apache Spark",
"content": " ",
"url": "/archives/v1.4.0/getting-started#getting-started-with-gluten-for-apache-spark",
"relUrl": "/archives/v1.4.0/getting-started#getting-started-with-gluten-for-apache-spark"
- },"716": {
+ },"718": {
"doc": "Getting-Started",
"title": "Getting-Started",
"content": " ",
"url": "/archives/v1.4.0/getting-started",
"relUrl": "/archives/v1.4.0/getting-started"
- },"717": {
+ },"719": {
"doc": "v1.4.0",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.4.0/#gluten-documents-by-version",
"relUrl": "/archives/v1.4.0/#gluten-documents-by-version"
- },"718": {
+ },"720": {
"doc": "v1.4.0",
"title": "v1.4.0",
"content": " ",
"url": "/archives/v1.4.0/",
"relUrl": "/archives/v1.4.0/"
- },"719": {
+ },"721": {
"doc": "Velox Backend",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.4.0/velox-backend#gluten-documents-by-version",
"relUrl": "/archives/v1.4.0/velox-backend#gluten-documents-by-version"
- },"720": {
+ },"722": {
"doc": "Velox Backend",
"title": "Velox Backend",
"content": " ",
"url": "/archives/v1.4.0/velox-backend",
"relUrl": "/archives/v1.4.0/velox-backend"
- },"721": {
+ },"723": {
"doc": "velox backend limitations",
"title": "velox backend limitations",
"content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-limitations.html",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-limitations.html"
- },"722": {
+ },"724": {
"doc": "velox backend scalar function support",
"title": "Scalar Functions Support Status",
"content": "Out of 357 scalar functions in Spark 3.5, Gluten currently fully supports 229 functions and partially supports 13 functions. ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#scalar-functions-support-status",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#scalar-functions-support-status"
- },"723": {
+ },"725": {
"doc": "velox backend scalar function support",
"title": "Array Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | array | CreateArray | S | | . | array_append | ArrayAppend | | | . | array_compact | ArrayCompact | | | . | array_contains | ArrayContains | S | | . | array_distinct | ArrayDistinct | S | | . | array_except | ArrayExcept | S | | . | array_insert | ArrayInsert | S | | . | array_intersect | ArrayIntersect | S | | . | array_join | ArrayJoin | S | | . | array_max | ArrayMax | S | | . | [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#array-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#array-functions"
- },"724": {
+ },"726": {
"doc": "velox backend scalar function support",
"title": "Bitwise Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | & | BitwiseAnd | S | | . | ^ | BitwiseXor | S | | . | bit_count | BitwiseCount | S | | . | bit_get | BitwiseGet | S | | . | getbit | BitwiseGet | S | | . | shiftright | ShiftRight | S | | . | shiftrightunsigned | ShiftRightUnsigned | S | | . | | BitwiseOr | S | | . | ~ | BitwiseNot | S | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#bitwise-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#bitwise-functions"
- },"725": {
+ },"727": {
"doc": "velox backend scalar function support",
"title": "Collection Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | array_size | ArraySize | | | . | cardinality | Size | S | | . | concat | Concat | S | | . | reverse | Reverse | S | | . | size | Size | S | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#collection-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#collection-functions"
- },"726": {
+ },"728": {
"doc": "velox backend scalar function support",
"title": "Conditional Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | coalesce | Coalesce | S | | . | if | If | S | | . | ifnull | Nvl | | | . | nanvl | NaNvl | S | | . | nullif | NullIf | | | . | nvl | Nvl | | | . | nvl2 | Nvl2 | | | . | when | CaseWhen | S | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#conditional-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#conditional-functions"
- },"727": {
+ },"729": {
"doc": "velox backend scalar function support",
"title": "Conversion Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | bigint | | S | | . | binary | | S | | . | boolean | | S | | . | cast | Cast | S | | . | date | | S | | . | decimal | | S | | . | double | | S | | . | float | | S | | . | int | | S | | . | smallint | | S | | . | string | | S | | . | timestamp | | S | | . | tinyint | | S | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#conversion-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#conversion-functions"
- },"728": {
+ },"730": {
"doc": "velox backend scalar function support",
"title": "Csv Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | from_csv | CsvToStructs | | | . | schema_of_csv | SchemaOfCsv | | | . | to_csv | StructsToCsv | | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#csv-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#csv-functions"
- },"729": {
+ },"731": {
"doc": "velox backend scalar function support",
"title": "Date and Timestamp Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | add_months | AddMonths | S | | . | convert_timezone | ConvertTimezone | | | . | curdate | CurDateExpressionBuilder | | | . | current_date | CurrentDate | | | . | current_timestamp | CurrentTimestamp | | | . | current_timezone | CurrentTimeZone | | | . | date_add | DateAdd | S | | . | date_diff | DateDiff | S | | . | date_format | DateFormatClass | S | | . | date_from_unix_date [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#date-and-timestamp-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#date-and-timestamp-functions"
- },"730": {
+ },"732": {
"doc": "velox backend scalar function support",
"title": "Hash Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | crc32 | Crc32 | S | | . | hash | Murmur3Hash | S | | . | md5 | Md5 | S | | . | sha | Sha1 | S | | . | sha1 | Sha1 | S | | . | sha2 | Sha2 | S | | . | xxhash64 | XxHash64 | S | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#hash-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#hash-functions"
- },"731": {
+ },"733": {
"doc": "velox backend scalar function support",
"title": "JSON Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | from_json | JsonToStructs | | | . | get_json_object | GetJsonObject | S | | . | json_array_length | LengthOfJsonArray | S | | . | json_object_keys | JsonObjectKeys | | | . | json_tuple | JsonTuple | S | | . | schema_of_json | SchemaOfJson | | | . | to_json | StructsToJson | | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#json-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#json-functions"
- },"732": {
+ },"734": {
"doc": "velox backend scalar function support",
"title": "Lambda Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | aggregate | ArrayAggregate | S | | . | array_sort | ArraySort | S | | . | exists | ArrayExists | S | | . | filter | ArrayFilter | S | | . | forall | ArrayForAll | S | | . | map_filter | MapFilter | | | . | map_zip_with | MapZipWith | S | | . | reduce | ArrayAggregate | S | | . | transform | ArrayTransform | S | | . | transform_keys | TransformKeys | S | | . | transform_values | Trans [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#lambda-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#lambda-functions"
- },"733": {
+ },"735": {
"doc": "velox backend scalar function support",
"title": "Map Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | element_at | ElementAt | S | | . | map | CreateMap | PS | | . | map_concat | MapConcat | | | . | map_contains_key | MapContainsKey | | | . | map_entries | MapEntries | S | | . | map_from_arrays | MapFromArrays | S | | . | map_from_entries | MapFromEntries | | | . | map_keys | MapKeys | S | | . | map_values | MapValues | S | | . | str_to_map | StringToMap | S | | . | try_element_a [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#map-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#map-functions"
- },"734": {
+ },"736": {
"doc": "velox backend scalar function support",
"title": "Mathematical Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | % | Remainder | S | | . | * | Multiply | S | | . | + | Add | S | | . | - | Subtract | S | | . | / | Divide | S | | . | abs | Abs | S | | . | acos | Acos | S | | . | acosh | Acosh | S | | . | asin | Asin | S | | . | asinh | Asinh | S | | . | atan | Atan | S | | . | atan2 | Atan2 | S | | . | atanh | Atanh | S | | . | bin | Bin | S | | . | bround | BRound | | | . | cbrt | Cbrt [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#mathematical-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#mathematical-functions"
- },"735": {
+ },"737": {
"doc": "velox backend scalar function support",
"title": "Misc Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | aes_decrypt | AesDecrypt | | | . | aes_encrypt | AesEncrypt | | | . | assert_true | AssertTrue | | | . | bitmap_bit_position | BitmapBitPosition | | | . | bitmap_bucket_number | BitmapBucketNumber | | | . | bitmap_count | BitmapCount | | | . | current_catalog | CurrentCatalog | | | . | current_database | CurrentDatabase | | | . | current_schema | CurrentDatabase | | | . [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#misc-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#misc-functions"
- },"736": {
+ },"738": {
"doc": "velox backend scalar function support",
"title": "Predicate Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | ! | Not | S | | . | != | | | | . | < | LessThan | S | | . | <= | LessThanOrEqual | S | | . | <=> | EqualNullSafe | S | | . | <> | | | | . | = | EqualTo | S | | . | == | EqualTo | S | | . | > | GreaterThan | S | | . | >= | GreaterThanOrEqual | S | | . | and | And | S | | . | between | | | | . | case | | | | . | ilike | ILike | | | . | in [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#predicate-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#predicate-functions"
- },"737": {
+ },"739": {
"doc": "velox backend scalar function support",
"title": "String Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | ascii | Ascii | S | | . | base64 | Base64 | S | | . | bit_length | BitLength | S | | . | btrim | StringTrimBoth | | | . | char | Chr | S | | . | char_length | Length | S | | . | character_length | Length | S | | . | chr | Chr | S | | . | concat_ws | ConcatWs | S | | . | contains | ContainsExpressionBuilder | S | | . | decode | Decode | | | . | elt | Elt | | | . | encode | Enc [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#string-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#string-functions"
- },"738": {
+ },"740": {
"doc": "velox backend scalar function support",
"title": "Struct Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | named_struct | CreateNamedStruct | S | | . | struct | | S | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#struct-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#struct-functions"
- },"739": {
+ },"741": {
"doc": "velox backend scalar function support",
"title": "URL Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | parse_url | ParseUrl | | | . | url_decode | UrlDecode | PS | | . | url_encode | UrlEncode | PS | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#url-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#url-functions"
- },"740": {
+ },"742": {
"doc": "velox backend scalar function support",
"title": "XML Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | xpath | XPathList | | | . | xpath_boolean | XPathBoolean | | | . | xpath_double | XPathDouble | | | . | xpath_float | XPathFloat | | | . | xpath_int | XPathInt | | | . | xpath_long | XPathLong | | | . | xpath_number | XPathDouble | | | . | xpath_short | XPathShort | | | . | xpath_string | XPathString | | | . ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#xml-functions",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html#xml-functions"
- },"741": {
+ },"743": {
"doc": "velox backend scalar function support",
"title": "velox backend scalar function support",
"content": " ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html"
- },"742": {
+ },"744": {
"doc": "velox backend support progress",
"title": "The Operators and Functions Support Progress",
"content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. We use some notations t [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-support-progress.html#the-operators-and-functions-support-progress",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-support-progress.html#the-operators-and-functions-support-progress"
- },"743": {
+ },"745": {
"doc": "velox backend support progress",
"title": "Operator Map",
"content": "Gluten supports 30+ operators (Drag to right to see all data types) . | Executor | Description | Gluten Name | Velox Name | BOOLEAN | BYTE | SHORT | INT | LONG | FLOAT | DOUBLE | STRING | NULL | BINARY | ARRAY | MAP | STRUCT(ROW) | DATE | TIMESTAMP | DECIMAL | CALENDAR | UDT | . | FileSourceScanExec | Reading data from files, often from Hive tables | FileSourceScanExecTransformer | TableScanNode | S | S | S | S | S | S | S | S | S | S | NS | NS | NS | S | NS | NS | NS | N [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-support-progress.html#operator-map",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-support-progress.html#operator-map"
- },"744": {
+ },"746": {
"doc": "velox backend support progress",
"title": "Function Support Status",
"content": "Spark categorizes built-in functions into four types: Scalar Functions, Aggregate Functions, Window Functions, and Generator Functions. In Gluten, function support is automatically generated by a script and maintained in separate files. Run the following command to generate and update the support status. Note that --spark_home should be set to the directory containing the Spark source code for the latest supported Spark version in Gluten, and the Spark project must be bui [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-support-progress.html#function-support-status",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-support-progress.html#function-support-status"
- },"745": {
+ },"747": {
"doc": "velox backend support progress",
"title": "velox backend support progress",
"content": " ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-support-progress.html",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-support-progress.html"
- },"746": {
+ },"748": {
"doc": "velox backend troubleshooting",
"title": "Troubleshooting",
"content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
"url": "/docs/v1.4.0/velox-backend/velox-backend-troubleshooting.html#troubleshooting",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-troubleshooting.html#troubleshooting"
- },"747": {
+ },"749": {
"doc": "velox backend troubleshooting",
"title": "velox backend troubleshooting",
"content": " ",
"url": "/docs/v1.4.0/velox-backend/velox-backend-troubleshooting.html",
"relUrl": "/docs/v1.4.0/velox-backend/velox-backend-troubleshooting.html"
- },"748": {
+ },"750": {
"doc": "Configuration",
"title": "Spark base configurations for Gluten plugin",
"content": "| Key | Recommend Setting | Description | . | spark.plugins | org.apache.gluten.GlutenPlugin | To load Gluten’s components by Spark’s plug-in loader. | . | spark.memory.offHeap.enabled | true | Gluten use off-heap memory for certain operations. | . | spark.memory.offHeap.size | 30G | The absolute amount of memory which can be used for off-heap allocation, in bytes unless otherwise specified. Note: Gluten Plugin will leverage this setting to allocate memory space for nativ [...]
"url": "/docs/v1.5.0/Configuration.html#spark-base-configurations-for-gluten-plugin",
"relUrl": "/docs/v1.5.0/Configuration.html#spark-base-configurations-for-gluten-plugin"
- },"749": {
+ },"751": {
"doc": "Configuration",
"title": "Gluten configurations",
"content": "| Key | Default | Description | . | spark.gluten.enabled | true | Whether to enable gluten. Default value is true. Just an experimental property. Recommend to enable/disable Gluten through the setting for spark.plugins. | . | spark.gluten.execution.resource.expired.time | 86400 | Expired time of execution with resource relation has cached. | . | spark.gluten.expression.blacklist | <undefined> | A black list of expression to skip transform, multiple values separated [...]
"url": "/docs/v1.5.0/Configuration.html#gluten-configurations",
"relUrl": "/docs/v1.5.0/Configuration.html#gluten-configurations"
- },"750": {
+ },"752": {
"doc": "Configuration",
"title": "Gluten experimental configurations",
"content": "| Key | Default | Description | . | spark.gluten.auto.adjustStageResource.enabled | false | Experimental: If enabled, gluten will try to set the stage resource according to stage execution plan. Only worked when aqe is enabled at the same time!! | . | spark.gluten.auto.adjustStageResources.fallenNode.ratio.threshold | 0.5 | Experimental: Increase executor heap memory when stage contains fallen node count exceeds the total node count ratio. | . | spark.gluten.auto.adjustSt [...]
"url": "/docs/v1.5.0/Configuration.html#gluten-experimental-configurations",
"relUrl": "/docs/v1.5.0/Configuration.html#gluten-experimental-configurations"
- },"751": {
+ },"753": {
"doc": "Configuration",
"title": "Configuration",
"content": " ",
"url": "/docs/v1.5.0/Configuration.html",
"relUrl": "/docs/v1.5.0/Configuration.html"
- },"752": {
+ },"754": {
"doc": "CppCodingStyle",
"title": "Gluten CPP Core Guidelines",
"content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#gluten-cpp-core-guidelines",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#gluten-cpp-core-guidelines"
- },"753": {
+ },"755": {
"doc": "CppCodingStyle",
"title": "Philosophy",
"content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#philosophy",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#philosophy"
- },"754": {
+ },"756": {
"doc": "CppCodingStyle",
"title": "Code Formatting",
"content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). | Always ensure your code is compatible with clang-format-15 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . To format cmake files like CMakeLists.txt & *.cmake, cmake-format is required to be installed. Here is an example. apt install python3-pip -y pip3 install --use [...]
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#code-formatting",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#code-formatting"
- },"755": {
+ },"757": {
"doc": "CppCodingStyle",
"title": "Naming Conventions",
"content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#naming-conventions",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#naming-conventions"
- },"756": {
+ },"758": {
"doc": "CppCodingStyle",
"title": "Designs",
"content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#designs",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#designs"
- },"757": {
+ },"759": {
"doc": "CppCodingStyle",
"title": "Source File & Header File",
"content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#source-file--header-file",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#source-file--header-file"
- },"758": {
+ },"760": {
"doc": "CppCodingStyle",
"title": "Class",
"content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#class",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#class"
- },"759": {
+ },"761": {
"doc": "CppCodingStyle",
"title": "Function",
"content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#function",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#function"
- },"760": {
+ },"762": {
"doc": "CppCodingStyle",
"title": "Variable",
"content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#variable",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#variable"
- },"761": {
+ },"763": {
"doc": "CppCodingStyle",
"title": "Constant",
"content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#constant",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#constant"
- },"762": {
+ },"764": {
"doc": "CppCodingStyle",
"title": "Macro",
"content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#macro",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#macro"
- },"763": {
+ },"765": {
"doc": "CppCodingStyle",
"title": "Namespace",
"content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#namespace",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#namespace"
- },"764": {
+ },"766": {
"doc": "CppCodingStyle",
"title": "Resource Management",
"content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#resource-management",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#resource-management"
- },"765": {
+ },"767": {
"doc": "CppCodingStyle",
"title": "Exception",
"content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#exception",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#exception"
- },"766": {
+ },"768": {
"doc": "CppCodingStyle",
"title": "Code Comment",
"content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#code-comment",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#code-comment"
- },"767": {
+ },"769": {
"doc": "CppCodingStyle",
"title": "References",
"content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html#references",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html#references"
- },"768": {
+ },"770": {
"doc": "CppCodingStyle",
"title": "CppCodingStyle",
"content": " ",
"url": "/docs/v1.5.0/developers/CppCodingStyle.html",
"relUrl": "/docs/v1.5.0/developers/CppCodingStyle.html"
- },"769": {
+ },"771": {
"doc": "HowTo",
"title": "How to understand the key work of Gluten?",
"content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsible for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the direct [...]
"url": "/docs/v1.5.0/developers/HowTo.html#how-to-understand-the-key-work-of-gluten",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#how-to-understand-the-key-work-of-gluten"
- },"770": {
+ },"772": {
"doc": "HowTo",
"title": "How to debug in Gluten?",
"content": " ",
"url": "/docs/v1.5.0/developers/HowTo.html#how-to-debug-in-gluten",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#how-to-debug-in-gluten"
- },"771": {
+ },"773": {
"doc": "HowTo",
"title": "1 How to debug C++",
"content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | Build Velox and Gluten CPP: | . ${GLUTEN_HOME}/dev/builddeps-velox [...]
"url": "/docs/v1.5.0/developers/HowTo.html#1-how-to-debug-c",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#1-how-to-debug-c"
- },"772": {
+ },"774": {
"doc": "HowTo",
"title": "2 How to debug plan validation process",
"content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in ${GLUTEN_HOME}/cpp/build/velox/benchmarks/. | R [...]
"url": "/docs/v1.5.0/developers/HowTo.html#2-how-to-debug-plan-validation-process",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#2-how-to-debug-plan-validation-process"
- },"773": {
+ },"775": {
"doc": "HowTo",
"title": "3 How to debug Java/Scala",
"content": "To debug some runtime issues in Scala/Java, we recommend developers to use Intellij remote debug, see tutorial link. According to your setting for Intellij remote debug, please set SPARK_SUBMIT_OPTS in the environment where spark-submit is executed. See the below example. export SPARK_SUBMIT_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8008 . To run a Scala/Java test class, you can use the below mvn command (take Velox backend as example), which is h [...]
"url": "/docs/v1.5.0/developers/HowTo.html#3-how-to-debug-javascala",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#3-how-to-debug-javascala"
- },"774": {
+ },"776": {
"doc": "HowTo",
"title": "4 How to debug with core-dump",
"content": "wait to complete . cd the_directory_of_core_file_generated gdb ${GLUTEN_HOME}/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
"url": "/docs/v1.5.0/developers/HowTo.html#4-how-to-debug-with-core-dump",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#4-how-to-debug-with-core-dump"
- },"775": {
+ },"777": {
"doc": "HowTo",
"title": "How to use jemalloc for Gluten native engine",
"content": "Currently, we have no dedicated memory allocator implemented by jemalloc. User can set environment variable LD_PRELOAD for lib jemalloc to let it override the corresponding C standard functions entirely. It may help alleviate OOM issues. spark.executorEnv.LD_PREALOD=/path/to/libjemalloc.so . ",
"url": "/docs/v1.5.0/developers/HowTo.html#how-to-use-jemalloc-for-gluten-native-engine",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#how-to-use-jemalloc-for-gluten-native-engine"
- },"776": {
+ },"778": {
"doc": "HowTo",
"title": "How to run TPC-H on Velox backend",
"content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of ${GLUTEN_HOME}/backends-velox/workload/tpch. The file README.md under ${GLUTEN_HOME}/backends-velox/workload/tpch offers some useful help, but it’s still not enough and exact. One way of run TPC-H test is to run velox-be by workflow, you can refer to velox_backend.yml . Here we will explain how to run TPC-H on Velox backend with the Parquet file format. | First, pre [...]
"url": "/docs/v1.5.0/developers/HowTo.html#how-to-run-tpc-h-on-velox-backend",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#how-to-run-tpc-h-on-velox-backend"
- },"777": {
+ },"779": {
"doc": "HowTo",
"title": "How to run TPC-DS",
"content": "wait to add . ",
"url": "/docs/v1.5.0/developers/HowTo.html#how-to-run-tpc-ds",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#how-to-run-tpc-ds"
- },"778": {
+ },"780": {
"doc": "HowTo",
"title": "How to track the memory exhaust problem",
"content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.memory.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of mem [...]
"url": "/docs/v1.5.0/developers/HowTo.html#how-to-track-the-memory-exhaust-problem",
"relUrl": "/docs/v1.5.0/developers/HowTo.html#how-to-track-the-memory-exhaust-problem"
- },"779": {
+ },"781": {
"doc": "HowTo",
"title": "HowTo",
"content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. When you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
"url": "/docs/v1.5.0/developers/HowTo.html",
"relUrl": "/docs/v1.5.0/developers/HowTo.html"
- },"780": {
+ },"782": {
"doc": "MicroBenchmarks",
"title": "Generate Micro Benchmarks for Velox Backend",
"content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#generate-micro-benchmarks-for-velox-backend",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#generate-micro-benchmarks-for-velox-backend"
- },"781": {
+ },"783": {
"doc": "MicroBenchmarks",
"title": "Try the example",
"content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3. mvn test -Pspark-3.2 -Pbackends-velox -pl backends-velox [...]
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#try-the-example",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#try-the-example"
- },"782": {
+ },"784": {
"doc": "MicroBenchmarks",
"title": "Generate Substrait plan and input for any query",
"content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#generate-substrait-plan-and-input-for-any-query",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#generate-substrait-plan-and-input-for-any-query"
- },"783": {
+ },"785": {
"doc": "MicroBenchmarks",
"title": "Save output to parquet for analysis",
"content": "You can save the output to a parquet file via --save-output <output> . Note: 1. This option cannot be used together with --with-shuffle. 2. This option cannot be used for write tasks. Please refer to section Simulate write tasks for more details. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan_{stageId}_{partitionId}_{vId}.json \\ --data /absolute_path/to/data_{stageId}_{partitionId}_{vId}_{iteratorIdx}.parquet --th [...]
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#save-output-to-parquet-for-analysis",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#save-output-to-parquet-for-analysis"
- },"784": {
+ },"786": {
"doc": "MicroBenchmarks",
"title": "Add shuffle write process",
"content": "You can add the shuffle write process at the end of the pipeline via --with-shuffle . Note: 1. This option cannot be used together with --save-output. 2. This option cannot be used for write tasks. Please refer to section Simulate write tasks for more details. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan_{stageId}_{partitionId}_{vId}.json \\ --split /absolute_path/to/split_{stageId}_{partitionId}_{vId}_{splitIdx}.json [...]
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#add-shuffle-write-process",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#add-shuffle-write-process"
- },"785": {
+ },"787": {
"doc": "MicroBenchmarks",
"title": "Enable debug mode",
"content": "spark.gluten.sql.debug(debug mode) is set to false by default thereby the google glog levels are limited to only print WARNING or higher severity logs. Unless spark.gluten.sql.debug is set in the INI file via --conf, the logging behavior is same as debug mode off. Developers can use --debug-mode command line flag to turn on debug mode when needed, and set verbosity/severity level via command line flags --v and --minloglevel. Note that constructing and deconstructing log s [...]
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#enable-debug-mode",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#enable-debug-mode"
- },"786": {
+ },"788": {
"doc": "MicroBenchmarks",
"title": "Enable HDFS support",
"content": "After enabling the dynamic loading of libhdfs.so at runtime to support HDFS, if you run the benchmark with an HDFS file, you need to set the classpath for Hadoop. You can do this by running . export CLASSPATH=`$HADOOP_HOME/bin/hdfs classpath --glob` . Otherwise, the HDFS connection will fail. If you have replaced ${HADOOP_HOME}/lib/native/libhdfs.so with libhdfs3.so, there is no need to set the CLASSPATH. ",
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#enable-hdfs-support",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#enable-hdfs-support"
- },"787": {
+ },"789": {
"doc": "MicroBenchmarks",
"title": "Simulate write tasks",
"content": "The last operator for a write task is a file write operator, and the output from Velox pipeline only contains several columns of statistics data. Therefore, specifying options --with-shuffle and --save-output does not take effect. You can specify the output path for the writer via --write-path option. Default is /tmp. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --write-p [...]
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#simulate-write-tasks",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#simulate-write-tasks"
- },"788": {
+ },"790": {
"doc": "MicroBenchmarks",
"title": "Simulate task spilling",
"content": "You can simulate task spilling by specify a memory hard limit from --memory_limit. By default, spilled files are written to the /tmp directory. To simulate real Gluten workloads, which utilize multiple spill directories, set the environment variable GLUTEN_SPARK_LOCAL_DIRS to a comma-separated string. Please check Simulate Gluten workload with multiple processes and threads for more details. ",
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#simulate-task-spilling",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#simulate-task-spilling"
- },"789": {
+ },"791": {
"doc": "MicroBenchmarks",
"title": "Simulate Gluten workload with multiple processes and threads",
"content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a bare-metal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --pla [...]
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html#simulate-gluten-workload-with-multiple-processes-and-threads",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html#simulate-gluten-workload-with-multiple-processes-and-threads"
- },"790": {
+ },"792": {
"doc": "MicroBenchmarks",
"title": "MicroBenchmarks",
"content": " ",
"url": "/docs/v1.5.0/developers/MicroBenchmarks.html",
"relUrl": "/docs/v1.5.0/developers/MicroBenchmarks.html"
- },"791": {
+ },"793": {
"doc": "NewToGluten",
"title": "Guide for New Developers",
"content": " ",
"url": "/docs/v1.5.0/developers/NewToGluten.html#guide-for-new-developers",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html#guide-for-new-developers"
- },"792": {
+ },"794": {
"doc": "NewToGluten",
"title": "Environment",
"content": "Gluten supports Ubuntu 20.04/22.04, CentOS 7/8, and MacOS. JDK . Gluten supports JDK 8 for Spark 3.2, 3.3, 3.4, and 3.5. For Spark 3.3 and later versions, Gluten also supports JDK 11 and 17. Note: Starting with Spark 4.0, the minimum required JDK version is 17. We recommend using a higher JDK version now to ease migration when deploying Gluten for Spark 4.0 in the future. In addition, we may upgrade Arrow from 15.0.0 to a newer release, which will require JDK 11 as the mi [...]
"url": "/docs/v1.5.0/developers/NewToGluten.html#environment",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html#environment"
- },"793": {
+ },"795": {
"doc": "NewToGluten",
"title": "Development",
"content": "To debug Java/Scala code, follow the steps in build-gluten-with-velox-backend. To debug C++ code, compile the backend code and gluten C++ code in debug mode. ## compile Velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . Note: To debug the tests under <gluten_home>/gluten-ut/, you must compile java code with -Pspark-ut. Java/scala code development . Linux IntelliJ local debu [...]
"url": "/docs/v1.5.0/developers/NewToGluten.html#development",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html#development"
- },"794": {
+ },"796": {
"doc": "NewToGluten",
"title": "Debug C++ Code with Core Dump",
"content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . core-Executor task l-2000883-1671542526 is the generated core [...]
"url": "/docs/v1.5.0/developers/NewToGluten.html#debug-c-code-with-core-dump",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html#debug-c-code-with-core-dump"
- },"795": {
+ },"797": {
"doc": "NewToGluten",
"title": "Debug C++ with GDB",
"content": "You can use GDB to debug tests, benchmarks, and JNI calls. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute GDB command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <velox_home>/velox/sub [...]
"url": "/docs/v1.5.0/developers/NewToGluten.html#debug-c-with-gdb",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html#debug-c-with-gdb"
- },"796": {
+ },"798": {
"doc": "NewToGluten",
"title": "Debug Memory Leaks",
"content": "Arrow memory allocator leak . If you receive an error message like the following: . 4/04/18 08:15:38 WARN ArrowBufferAllocators$ArrowBufferAllocatorManager: Detected leaked Arrow allocator [Default], size: 191, process accumulated leaked size: 191... 24/04/18 08:15:38 WARN ArrowBufferAllocators$ArrowBufferAllocatorManager: Leaked allocator stack Allocator(ROOT) 0/191/319/9223372036854775807 (res/actual/peak/limit) . You can open the Arrow allocator debug config by adding [...]
"url": "/docs/v1.5.0/developers/NewToGluten.html#debug-memory-leaks",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html#debug-memory-leaks"
- },"797": {
+ },"799": {
"doc": "NewToGluten",
"title": "Run TPC-H and TPC-DS",
"content": "We supply <gluten_home>/tools/gluten-it to execute these queries. See velox_backend_x86.yml. ",
"url": "/docs/v1.5.0/developers/NewToGluten.html#run-tpc-h-and-tpc-ds",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html#run-tpc-h-and-tpc-ds"
- },"798": {
+ },"800": {
"doc": "NewToGluten",
"title": "Enable Gluten for Spark",
"content": "To enable Gluten Velox backend for Spark, run the following command: . spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=org.apache.gluten.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://dlcdn.apache.org/incubator/gluten/1.4.0-incubating/apache-gluten-1.4.0-incubating-bin-spark35.tar.gz \\ --conf spark.shuffle.manager=org.apache.spark.shuffle.sort.ColumnarShuffleMa [...]
"url": "/docs/v1.5.0/developers/NewToGluten.html#enable-gluten-for-spark",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html#enable-gluten-for-spark"
- },"799": {
+ },"801": {
"doc": "NewToGluten",
"title": "Gluten Plan Validation and Updates",
"content": "VeloxTPCHSuite can verify the executed Gluten plans for the TPC-H benchmark to avoid unintentional changes. This verification is based on comparisons with the golden files that record the expected Gluten plans. The following failure may occur in GitHub CI or local tests: . - TPC-H q5 *** FAILED *** Mismatch for query 5 Actual Plan path: /tmp/tpch-approved-plan/v2-bhj/spark322/5.txt Golden Plan path: /opt/gluten/backends-velox/target/scala-2.12/test-classes/tpch-approved-p [...]
"url": "/docs/v1.5.0/developers/NewToGluten.html#gluten-plan-validation-and-updates",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html#gluten-plan-validation-and-updates"
- },"800": {
+ },"802": {
"doc": "NewToGluten",
"title": "NewToGluten",
"content": " ",
"url": "/docs/v1.5.0/developers/NewToGluten.html",
"relUrl": "/docs/v1.5.0/developers/NewToGluten.html"
- },"801": {
+ },"803": {
"doc": "PartialProject",
"title": "Partial Projection Support",
"content": "In Gluten, there is still a gap in supporting all Spark expressions natively (e.g., some JSON functions or Java UDFs). In this case, Gluten will choose the JVM code path to run the expressions, which can introduce performance regressions. Partial projections, which allow Gluten to minimal data copy between JVM and C++, were added to avoid these performance regressions. ",
"url": "/docs/v1.5.0/developers/PartialProject.html#partial-projection-support",
"relUrl": "/docs/v1.5.0/developers/PartialProject.html#partial-projection-support"
- },"802": {
+ },"804": {
"doc": "PartialProject",
"title": "Detailed Implementations",
"content": "Adding Partial Projection for UDF . For example, with the expression hash(udf(col0)), col1, col2, col3, col4, partial projection allows us to convert only col0 to row or column to Arrow as input, and convert udf(col0) as an alias partialProject1_. Then, ProjectExecTransformer will handle hash(partialProject1_), col1, col2, col3, col4, partialProject1_. This feature saves the cost of converting the columnar format to row format and vice-versa. ",
"url": "/docs/v1.5.0/developers/PartialProject.html#detailed-implementations",
"relUrl": "/docs/v1.5.0/developers/PartialProject.html#detailed-implementations"
- },"803": {
+ },"805": {
"doc": "PartialProject",
"title": "Adding Partial Projection for Unsupported Expressions",
"content": "The partial projection feature can also benefit from expressions that are not natively supported. For example, substr(from_json(col_a)). Since from_json is not fully supported, Gluten may use the JVM code path. Instead of projecting the whole expression, partial projection will attempt to project from_json() and perform a native projection of substr(). In the case of blacklisted expressions defined in spark.gluten.expression.blacklist, this feature is also beneficial. ",
"url": "/docs/v1.5.0/developers/PartialProject.html#adding-partial-projection-for-unsupported-expressions",
"relUrl": "/docs/v1.5.0/developers/PartialProject.html#adding-partial-projection-for-unsupported-expressions"
- },"804": {
+ },"806": {
"doc": "PartialProject",
"title": "Limitations",
"content": "This feature is in the preliminary stages of development and will be improved in future updates. ",
"url": "/docs/v1.5.0/developers/PartialProject.html#limitations",
"relUrl": "/docs/v1.5.0/developers/PartialProject.html#limitations"
- },"805": {
+ },"807": {
"doc": "PartialProject",
"title": "PartialProject",
"content": " ",
"url": "/docs/v1.5.0/developers/PartialProject.html",
"relUrl": "/docs/v1.5.0/developers/PartialProject.html"
- },"806": {
+ },"808": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Profile with gperftools",
"content": "gperftools is a collection of a high-performance multi-threaded malloc() implementation, plus some pretty nifty performance analysis tools, see more: https://github.com/gperftools/gperftools/wiki. ",
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#profile-with-gperftools",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#profile-with-gperftools"
- },"807": {
+ },"809": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Build and install gperftools",
"content": "Download gperftools from https://github.com/gperftools/gperftools/releases, build and install. wget https://github.com/gperftools/gperftools/releases/download/gperftools<version>/gperftools-<version>.tar.gz tar xzvf gperftools-<version>.tar.gz cd gperftools-<version> ./configure make && make install . Then we can find the tcmalloc libraries in $GPERFTOOLS_HOME/.lib. ",
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#build-and-install-gperftools",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#build-and-install-gperftools"
- },"808": {
+ },"810": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Run Gluten with gperftools",
"content": "Configure --files or spark.files for Spark. --files /path/to/gperftools/libtcmalloc_and_profiler.so or spark.files /path/to/gperftools/libtcmalloc_and_profiler.so . Use LD_PRELOAD to preload tcmalloc library, and enable heap profile with HEAPPROFILE or cpu profile with CPUPROFILE. Example of enabling heap profile in spark executor: . spark.executorEnv.LD_PRELOAD ./libtcmalloc_and_profiler.so # Specifies dump profile path. ${CONTAINER_ID} is only used to distinguish the re [...]
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#run-gluten-with-gperftools",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#run-gluten-with-gperftools"
- },"809": {
+ },"811": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Analyze profiling output",
"content": "Prepare the required native libraries. Assume static build is used for Gluten, there is no other shared dependency libs. jar xf gluten-velox-bundle-spark3.5_2.12-centos_7_x86_64-1.2.0.jar relative/path/to/libvelox.so ralative/path/to/libgluten.so mv libvelox.so libgluten.so /path/to/gluten_lib_prefix . Generate a GIF of the analysis result: . # `/usr/bin/java` indicates the program used by running spark executor pprof --show_bytes --gif --lib_prefix=/path/to/gluten_lib_pr [...]
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#analyze-profiling-output",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#analyze-profiling-output"
- },"810": {
+ },"812": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Profile with jemalloc",
"content": "jemalloc is a general purpose malloc(3) implementation that emphasizes fragmentation avoidance and scalable concurrency support. We can also use it to analyze Gluten performance. Getting Started with jemalloc: https://github.com/jemalloc/jemalloc/wiki/Getting-Started. ",
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#profile-with-jemalloc",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#profile-with-jemalloc"
- },"811": {
+ },"813": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Build and install jemalloc",
"content": "Download jemalloc from https://github.com/jemalloc/jemalloc/releases, build and install. cd /path/to/jemalloc ./autogen.sh --enable-prof make && make install . Then we can find the jemalloc library in $JEMALLOC_HOME/.lib. ",
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#build-and-install-jemalloc",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#build-and-install-jemalloc"
- },"812": {
+ },"814": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Run Gluten with jemalloc",
"content": "Configure --files or spark.files for Spark. --files /path/to/jemalloc/libjemalloc.so or spark.files /path/to/jemalloc/libjemalloc.so . Example of enabling heap profile in spark executor: . spark.executorEnv.LD_PRELOAD ./libjemalloc.so spark.executorEnv.MALLOC_CONF prof:true,lg_prof_interval:30,prof_prefix:/tmp/gluten_heap_perf . Finally, profiling files prefixed with /tmp/gluten_heap_perf.${PID} will be generated for each spark executor. ",
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#run-gluten-with-jemalloc",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#run-gluten-with-jemalloc"
- },"813": {
+ },"815": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Memory dump on spark executor exit",
"content": "Sometimes, when native memory is not managed by gluten or there are some memory leaks that will cause spark executor to be killed due to memory limit, we only need to trigger a memory dump on executor exit. If we want to enable this feature we need to follow steps: . | Build gluten with --enable_jemalloc_stats=ON to enabled jemalloc stats. | Enabled memory dump on exit, add spark executor environments to load jemalloc lib and make memory profiling active. spark.gluten.mon [...]
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#memory-dump-on-spark-executor-exit",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#memory-dump-on-spark-executor-exit"
- },"814": {
+ },"816": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "Analyze profiling output",
"content": "Prepare the required native libraries. Assume static build is used for Gluten, so there is no other shared dependency libs. jar xf gluten-velox-bundle-spark3.5_2.12-centos_7_x86_64-1.2.0.jar relative/path/to/libvelox.so relative/path/to/libgluten.so mv libvelox.so libgluten.so /path/to/gluten_lib_prefix . Generate a GIF of the analysis result: . # `/usr/bin/java` indicates the program used by running spark executor jeprof --show_bytes --gif --lib_prefix=/path/to/gluten_li [...]
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#analyze-profiling-output-1",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html#analyze-profiling-output-1"
- },"815": {
+ },"817": {
"doc": "ProfileMemoryOfGlutenWithVelox",
"title": "ProfileMemoryOfGlutenWithVelox",
"content": "Gluten offloads most of Spark SQL execution to native engine. We can use gperftools or jemalloc to analyze the offheap memory and cpu profile. ",
"url": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html",
"relUrl": "/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html"
- },"816": {
+ },"818": {
"doc": "QueryTrace",
"title": "Background",
"content": "Currently, we have MicroBenchmarks to profile the Velox plan execution in stage level, now we have query trace replayer to specify plan node id to profile in operator level. We cannot profile operator level directly because ValueStreamNode cannot serialize and deserialize, so we should generate benchmark first, and then enable query trace in benchmark which replaces the ValueStreamNode to ValuesNode. Relevant link: https://facebookincubator.github.io/velox/develop/debuggi [...]
"url": "/docs/v1.5.0/developers/QueryTrace.html#background",
"relUrl": "/docs/v1.5.0/developers/QueryTrace.html#background"
- },"817": {
+ },"819": {
"doc": "QueryTrace",
"title": "Usage",
"content": "Firstly, we should use MicroBenchmark to generate the input parquet data file and substrait json plan. /tmp/saveDir ├── conf_35_0.ini ├── data_35_0_0.parquet ├── plan_35_0.json └── split_35_0_0.json . Secondly, execute the benchmark to get the task id and query trace node id, for example, the task id is Gluten_Stage_0_TID_0_VTID_0 and we will profile partial aggregation of node id 7. W0113 15:30:18.903225 2524110 GenericBenchmark.cc:251] Setting CPU for thread 0 to 0 I011 [...]
"url": "/docs/v1.5.0/developers/QueryTrace.html#usage",
"relUrl": "/docs/v1.5.0/developers/QueryTrace.html#usage"
- },"818": {
+ },"820": {
"doc": "QueryTrace",
"title": "QueryTrace",
"content": " ",
"url": "/docs/v1.5.0/developers/QueryTrace.html",
"relUrl": "/docs/v1.5.0/developers/QueryTrace.html"
- },"819": {
+ },"821": {
"doc": "SubstraitModifications",
"title": "Substrait Modifications in Gluten",
"content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
"url": "/docs/v1.5.0/developers/SubstraitModifications.html#substrait-modifications-in-gluten",
"relUrl": "/docs/v1.5.0/developers/SubstraitModifications.html#substrait-modifications-in-gluten"
- },"820": {
+ },"822": {
"doc": "SubstraitModifications",
"title": "Modifications to algebra.proto",
"content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
"url": "/docs/v1.5.0/developers/SubstraitModifications.html#modifications-to-algebraproto",
"relUrl": "/docs/v1.5.0/developers/SubstraitModifications.html#modifications-to-algebraproto"
- },"821": {
+ },"823": {
"doc": "SubstraitModifications",
"title": "Modifications to type.proto",
"content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
"url": "/docs/v1.5.0/developers/SubstraitModifications.html#modifications-to-typeproto",
"relUrl": "/docs/v1.5.0/developers/SubstraitModifications.html#modifications-to-typeproto"
- },"822": {
+ },"824": {
"doc": "SubstraitModifications",
"title": "SubstraitModifications",
"content": " ",
"url": "/docs/v1.5.0/developers/SubstraitModifications.html",
"relUrl": "/docs/v1.5.0/developers/SubstraitModifications.html"
- },"823": {
+ },"825": {
"doc": "UsingGperftoolsInCH",
"title": "Install gperftools",
"content": "Install gperftools as described in https://github.com/gperftools/gperftools. We get the library and the command line tools. ",
"url": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html#install-gperftools",
"relUrl": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html#install-gperftools"
- },"824": {
+ },"826": {
"doc": "UsingGperftoolsInCH",
"title": "Compiler libch.so",
"content": "Disable jemalloc -DENABLE_JEMALLOC=OFF in cpp-ch/CMakeLists.txt, and recompile libch.so. ",
"url": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html#compiler-libchso",
"relUrl": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html#compiler-libchso"
- },"825": {
+ },"827": {
"doc": "UsingGperftoolsInCH",
"title": "Run Gluten with gperftools",
"content": "For Spark on Yarn, we can change the submit script to run Gluten with gperftools. Add the following to the submit script: . export tcmalloc_path=/data2/zzb/gperftools-2.10/.libs/libtcmalloc_and_profiler.so # the path to the tcmalloc library export LD_PRELOAD=$tcmalloc_path,libch.so # load the library in the driver --files $tcmalloc_path # upload the library to the cluster --conf spark.executorEnv.LD_PRELOAD=./libtcmalloc_and_profiler.so,libch.so # load the library in the [...]
"url": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html#run-gluten-with-gperftools",
"relUrl": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html#run-gluten-with-gperftools"
- },"826": {
+ },"828": {
"doc": "UsingGperftoolsInCH",
"title": "Analyze the result",
"content": "We can get the result in the path we set in the previous step. For example, we can get the result in /tmp/gluten_heap_perf. We can use the following website to analyze the result: https://gperftools.github.io/gperftools/heapprofile.html https://gperftools.github.io/gperftools/cpuprofile.html . ",
"url": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html#analyze-the-result",
"relUrl": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html#analyze-the-result"
- },"827": {
+ },"829": {
"doc": "UsingGperftoolsInCH",
"title": "UsingGperftoolsInCH",
"content": "We need using gpertools to find the memory or CPU issue. That’s what this document is about. ",
"url": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html",
"relUrl": "/docs/v1.5.0/developers/UsingGperftoolsInCH.html"
- },"828": {
+ },"830": {
"doc": "UsingJemallocWithCH",
"title": "Change code of jemalloc",
"content": "Use libunwind instead of libgcc to get the backtrace which may cause process hangs. @@ -177,7 +177,8 @@ target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1) # At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracking. # # ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1). -target_compile_definitions (_jemalloc PRIV [...]
"url": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#change-code-of-jemalloc",
"relUrl": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#change-code-of-jemalloc"
- },"829": {
+ },"831": {
"doc": "UsingJemallocWithCH",
"title": "Get jeprof",
"content": "Change to the directory where you want to install jemalloc, and run the following commands: . cd $Clickhouse_SOURCE_PATH/contrib/jemalloc && ./autogen.sh && ./configure.sh && make -j8 . Then we get jeprof in the directory $Clickhouse_SOURCE_PATH/contrib/jemalloc/bin/jeprof. ",
"url": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#get-jeprof",
"relUrl": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#get-jeprof"
- },"830": {
+ },"832": {
"doc": "UsingJemallocWithCH",
"title": "Compiler libch.so",
"content": "Ensure to enable jemalloc -DENABLE_JEMALLOC=ON in cpp-ch/CMakeLists.txt, and compile libch.so. ",
"url": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#compiler-libchso",
"relUrl": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#compiler-libchso"
- },"831": {
+ },"833": {
"doc": "UsingJemallocWithCH",
"title": "Run Gluten with jemalloc heap tools",
"content": "For Yarn or thrift server, you need add the following to the submit script: . export MALLOC_CONF=\"prof:true,lg_prof_interval:30\" # enable jemalloc heap profiling . You can find more options in https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling. ",
"url": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#run-gluten-with-jemalloc-heap-tools",
"relUrl": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#run-gluten-with-jemalloc-heap-tools"
- },"832": {
+ },"834": {
"doc": "UsingJemallocWithCH",
"title": "Analyze the result",
"content": "After you get the heap file, you can use the following command to analyze the result: Check memory diff, so you can find which part of the code consume the memory. jeprof --svg /usr/lib/jvm/java-8-openjdk-amd64/bin/java --base=jeprof.3717358.1.i1.heap jeprof.3717358.30.i30.heap > diff.svg . You can find more usage about jeprof with jeprof --help. ",
"url": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#analyze-the-result",
"relUrl": "/docs/v1.5.0/developers/UsingJemallocWithCH.html#analyze-the-result"
- },"833": {
+ },"835": {
"doc": "UsingJemallocWithCH",
"title": "UsingJemallocWithCH",
"content": "We need using jemalloc to find the memory issue. That’s what this document is about. ",
"url": "/docs/v1.5.0/developers/UsingJemallocWithCH.html",
"relUrl": "/docs/v1.5.0/developers/UsingJemallocWithCH.html"
- },"834": {
+ },"836": {
"doc": "VeloxDynamicSizingOffheap",
"title": "Dynamic Off-heap Sizing",
"content": "Gluten requires setting both on-heap and off-heap memory sizes, which initializes different memory layouts. Improper configuration of these settings can lead to lower performance. To fix this issue, dynamic off-heap sizing is an experimental feature designed to simplify this process. When enabled, off-heap settings are ignored, and Velox uses the on-heap size as the memory size. ",
"url": "/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html#dynamic-off-heap-sizing",
"relUrl": "/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html#dynamic-off-heap-sizing"
- },"835": {
+ },"837": {
"doc": "VeloxDynamicSizingOffheap",
"title": "Detail implementations",
"content": "To understand the details, it’s essential to learn the basics of JVM memory management. There are many resources discussing JVM internals: . | https://exia.dev/blog/2019-12-10/JVM-Memory-Model/ | https://docs.oracle.com/cd/E13150_01/jrockit_jvm/jrockit/geninfo/diagnos/garbage_collect.html | https://www.scaler.com/topics/memory-management-in-java/ | https://developers.redhat.com/articles/2021/09/09/how-jvm-uses-and-allocates-memory# | . In general, the feature works as fol [...]
"url": "/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html#detail-implementations",
"relUrl": "/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html#detail-implementations"
- },"836": {
+ },"838": {
"doc": "VeloxDynamicSizingOffheap",
"title": "Limitations",
"content": "This feature is in the preliminary stages of development and will be improved in future updates. ",
"url": "/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html#limitations",
"relUrl": "/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html#limitations"
- },"837": {
+ },"839": {
"doc": "VeloxDynamicSizingOffheap",
"title": "VeloxDynamicSizingOffheap",
"content": " ",
"url": "/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html",
"relUrl": "/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html"
- },"838": {
+ },"840": {
"doc": "VeloxUDF",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": " ",
"url": "/docs/v1.5.0/developers/VeloxUDF.html#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"839": {
+ },"841": {
"doc": "VeloxUDF",
"title": "Introduction",
"content": "Velox backend supports User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF). Users can implement custom functions using the UDF interface provided by Velox and compile them into libraries. At runtime, these UDFs are registered alongside their Java implementations via CREATE TEMPORARY FUNCTION. Once registered, Gluten can parse and offload these UDFs to Velox during execution, meanwhile ensuring proper fallback to Java UDFs when necessary. ",
"url": "/docs/v1.5.0/developers/VeloxUDF.html#introduction",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html#introduction"
- },"840": {
+ },"842": {
"doc": "VeloxUDF",
"title": "Create and Build UDF/UDAF library",
"content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
"url": "/docs/v1.5.0/developers/VeloxUDF.html#create-and-build-udfudaf-library",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html#create-and-build-udfudaf-library"
- },"841": {
+ },"843": {
"doc": "VeloxUDF",
"title": "Using UDF/UDAF in Gluten",
"content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the library paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
"url": "/docs/v1.5.0/developers/VeloxUDF.html#using-udfudaf-in-gluten",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html#using-udfudaf-in-gluten"
- },"842": {
+ },"844": {
"doc": "VeloxUDF",
"title": "Try the example",
"content": "We provided Velox UDF examples in file MyUDF.cc and UDAF examples in file MyUDAF.cc. You need to build the gluten project with --build_example=ON to get the example libraries./dev/buildbundle-veloxbe.sh --build_examples=ON . Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ . Start spark-shell or spark-sql with below configuration . # Use the `--files` option to upload a library and configure its relative path --files /path/to/glute [...]
"url": "/docs/v1.5.0/developers/VeloxUDF.html#try-the-example",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html#try-the-example"
- },"843": {
+ },"845": {
"doc": "VeloxUDF",
"title": "Configurations",
"content": "| Parameters | Description | . | spark.gluten.sql.columnar.backend.velox.udfLibraryPaths | Path to the udf/udaf libraries. | . | spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths | Path to the udf/udaf libraries on driver node. Only applicable on yarn-client mode. | . | spark.gluten.sql.columnar.backend.velox.udfAllowTypeConversion | Whether to inject possible cast to convert mismatched data types from input to one registered signatures. | . ",
"url": "/docs/v1.5.0/developers/VeloxUDF.html#configurations",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html#configurations"
- },"844": {
+ },"846": {
"doc": "VeloxUDF",
"title": "Pandas UDFs (a.k.a. Vectorized UDFs)",
"content": " ",
"url": "/docs/v1.5.0/developers/VeloxUDF.html#pandas-udfs-aka-vectorized-udfs",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html#pandas-udfs-aka-vectorized-udfs"
- },"845": {
+ },"847": {
"doc": "VeloxUDF",
"title": "Introduction",
"content": "Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer doc. ",
"url": "/docs/v1.5.0/developers/VeloxUDF.html#introduction-1",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html#introduction-1"
- },"846": {
+ },"848": {
"doc": "VeloxUDF",
"title": "Using Pandas UDFs in Gluten with Velox Backend",
"content": "Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: . pip3 install pyspark==$SPARK_VERSION cython pip3 install pandas pyarrow . Gluten provides a config to control enable ColumnarArrowEvalPython or not, with true as default. spark.gluten.sql.columnar.arrowUdf . Then take following PySpark code for example: . from pyspark.sql.functions import pandas_udf, PandasUDFType import pyspark.sql.functions a [...]
"url": "/docs/v1.5.0/developers/VeloxUDF.html#using-pandas-udfs-in-gluten-with-velox-backend",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html#using-pandas-udfs-in-gluten-with-velox-backend"
- },"847": {
+ },"849": {
"doc": "VeloxUDF",
"title": "VeloxUDF",
"content": " ",
"url": "/docs/v1.5.0/developers/VeloxUDF.html",
"relUrl": "/docs/v1.5.0/developers/VeloxUDF.html"
- },"848": {
+ },"850": {
"doc": "clickhouse backend debug",
"title": "Debug CH Backend",
"content": " ",
"url": "/docs/v1.5.0/developers/clickhouse-backend-debug.html#debug-ch-backend",
"relUrl": "/docs/v1.5.0/developers/clickhouse-backend-debug.html#debug-ch-backend"
- },"849": {
+ },"851": {
"doc": "clickhouse backend debug",
"title": "Debug Java/Scala Code with IntelliJ IDEA",
"content": ". | Build Gluten ClickHouse Native Lib. export CMAKE_BUILD_TYPE=Release && bash ep/build-clickhouse/src/build_clickhouse.sh . libch.so will be generated in cpp-ch/build/utils/extern-local-engine/libch.so. | Maven Build Gluten ClickHouse with Profile mvn clean install -DskipTests -P delta -Pbackends-clickhouse -Pspark-3.3 -Pspark-ut . | Set Maven Profiles in IntelliJ IDEA . | Set Debug Configuration in IntelliJ IDEA . For example, debug GlutenMathExpressionsSuite. [...]
"url": "/docs/v1.5.0/developers/clickhouse-backend-debug.html#debug-javascala-code-with-intellij-idea",
"relUrl": "/docs/v1.5.0/developers/clickhouse-backend-debug.html#debug-javascala-code-with-intellij-idea"
- },"850": {
+ },"852": {
"doc": "clickhouse backend debug",
"title": "Debug Native Code with CLion",
"content": ". | Toolchains Settings . Some Clion versions have not supported lldb-18, you can manually set refer to CLion Custom LLDB. Or you can use gdb as default. | CMake Debug Configuration . CMake Options: -DENABLE_PROTOBUF=ON -DENABLE_TESTS=OFF -DENABLE_BENCHMARKS=ON -DENABLE_JEMALLOC=ON -DENABLE_MULTITARGET_CODE=ON -DENABLE_EXTERN_LOCAL_ENGINE=ON -DENABLE_ODBC=OFF -DENABLE_CAPNP=OFF -DENABLE_ROCKSDB=OFF -DENABLE_GRPC=OFF -DENABLE_RUST=OFF -DENABLE_H3=OFF -DENABLE_AMQPCPP=OFF - [...]
"url": "/docs/v1.5.0/developers/clickhouse-backend-debug.html#debug-native-code-with-clion",
"relUrl": "/docs/v1.5.0/developers/clickhouse-backend-debug.html#debug-native-code-with-clion"
- },"851": {
+ },"853": {
"doc": "clickhouse backend debug",
"title": "clickhouse backend debug",
"content": " ",
"url": "/docs/v1.5.0/developers/clickhouse-backend-debug.html",
"relUrl": "/docs/v1.5.0/developers/clickhouse-backend-debug.html"
- },"852": {
+ },"854": {
"doc": "improvement proposals",
"title": "Gluten Project Improvement Proposals (GPIP)",
"content": "The Gluten Project Improvement Proposals doc references the Spark SPIP documentation. The purpose of a GPIP is to inform and involve the user community in major improvements to the Gluten codebase throughout the development process to increase the likelihood that user needs are met. GPIPs should be used for significant user-facing or cutting-edge changes, not small incremental improvements. If your proposal meets the definition of GPIP, we recommend you to create a GPIP, [...]
"url": "/docs/v1.5.0/developers/improvement-proposals.html#gluten-project-improvement-proposals-gpip",
"relUrl": "/docs/v1.5.0/developers/improvement-proposals.html#gluten-project-improvement-proposals-gpip"
- },"853": {
+ },"855": {
"doc": "improvement proposals",
"title": "What is a GPIP?",
"content": "A GPIP is similar to a product requirement document commonly used in product management. A GPIP: . | Is a ticket labeled “GPIP” proposing a major improvement or change to Gluten | Follows the template defined below | Includes discussions on the ticket and dev@ list about the proposal | . ",
"url": "/docs/v1.5.0/developers/improvement-proposals.html#what-is-a-gpip",
"relUrl": "/docs/v1.5.0/developers/improvement-proposals.html#what-is-a-gpip"
- },"854": {
+ },"856": {
"doc": "improvement proposals",
"title": "Who?",
"content": "Any community member can help by discussing whether a GPIP is likely to meet their needs and propose GPIPs. Contributors can help by discussing whether a GPIP is likely to be technically feasible. Committers can help by discussing whether a GPIP aligns with long-term project goals, and by shepherding GPIPs. GPIP Author is any community member who authors a GPIP and is committed to pushing the change through the entire process. GPIP authorship can be transferred. GPIP Shep [...]
"url": "/docs/v1.5.0/developers/improvement-proposals.html#who",
"relUrl": "/docs/v1.5.0/developers/improvement-proposals.html#who"
- },"855": {
+ },"857": {
"doc": "improvement proposals",
"title": "GPIP Process",
"content": "Proposing a GPIP . Anyone may propose a GPIP, using the document template below. Please only submit a GPIP if you are willing to help, at least with discussion. After a GPIP is created, the author should email dev@gluten.apache.org to notify the community of the GPIP, and discussions should ensue on the ticket. If a GPIP is too small or incremental and should have been done through the normal ticket process, a committer should remove the GPIP label. GPIP Document Template [...]
"url": "/docs/v1.5.0/developers/improvement-proposals.html#gpip-process",
"relUrl": "/docs/v1.5.0/developers/improvement-proposals.html#gpip-process"
- },"856": {
+ },"858": {
"doc": "improvement proposals",
"title": "improvement proposals",
"content": " ",
"url": "/docs/v1.5.0/developers/improvement-proposals.html",
"relUrl": "/docs/v1.5.0/developers/improvement-proposals.html"
- },"857": {
+ },"859": {
"doc": "Developers",
"title": "Gluten Developers",
"content": "This document provides a developer overview of the project and covers the following topics: . ",
"url": "/archives/v1.5.0/developers#gluten-developers",
"relUrl": "/archives/v1.5.0/developers#gluten-developers"
- },"858": {
+ },"860": {
"doc": "Developers",
"title": "Developers",
"content": " ",
"url": "/archives/v1.5.0/developers",
"relUrl": "/archives/v1.5.0/developers"
- },"859": {
+ },"861": {
"doc": "velox backend CI",
"title": "Velox Backend CI",
"content": "GHA workflows are defined under .github/workflows/. ",
"url": "/docs/v1.5.0/developers/velox-backend-CI.html#velox-backend-ci",
"relUrl": "/docs/v1.5.0/developers/velox-backend-CI.html#velox-backend-ci"
- },"860": {
+ },"862": {
"doc": "velox backend CI",
"title": "Docker Build",
"content": "We have a weekly job defined in docker_image.yml to build docker images for CI verification. The docker files and images are listed below: . | file | images | comments | . | dev/docker/Dockerfile.centos7-static-build | apache/gluten:vcpkg-centos-7 | centos 7, static link, jdk8 | . | dev/docker/Dockerfile.centos8-static-build | apache/gluten:vcpkg-centos-8 | centos 8, static link, jdk8 | . | dev/docker/Dockerfile.centos8-dynamic-build | apache/gluten:centos-8-jdk8 | centos [...]
"url": "/docs/v1.5.0/developers/velox-backend-CI.html#docker-build",
"relUrl": "/docs/v1.5.0/developers/velox-backend-CI.html#docker-build"
- },"861": {
+ },"863": {
"doc": "velox backend CI",
"title": "Vcpkg Caching",
"content": "Gluten main branch is pulled down during static build in docker. And vcpkg will cache binary data of all dependencies defined under dev/vcpkg. These binary data is cached into /var/cache/vcpkg and CI job can re-use them in new build. By setting VCPKG_BINARY_SOURCES=clear in env., reusing vcpkg cache can be disabled. ",
"url": "/docs/v1.5.0/developers/velox-backend-CI.html#vcpkg-caching",
"relUrl": "/docs/v1.5.0/developers/velox-backend-CI.html#vcpkg-caching"
- },"862": {
+ },"864": {
"doc": "velox backend CI",
"title": "Arrow Libs Pre-installation",
"content": "Arrow libs are pre-installed in docker, assuming they are not actively changed, then not necessarily to be re-built every time. ",
"url": "/docs/v1.5.0/developers/velox-backend-CI.html#arrow-libs-pre-installation",
"relUrl": "/docs/v1.5.0/developers/velox-backend-CI.html#arrow-libs-pre-installation"
- },"863": {
+ },"865": {
"doc": "velox backend CI",
"title": ".M2 cache",
"content": "The dependency libraries are pre installed in to /root/.m2 by mvn dependency:go-offline Spark is set to 3.5 by default. ",
"url": "/docs/v1.5.0/developers/velox-backend-CI.html#m2-cache",
"relUrl": "/docs/v1.5.0/developers/velox-backend-CI.html#m2-cache"
- },"864": {
+ },"866": {
"doc": "velox backend CI",
"title": "Ccache",
"content": "Since the docker image is rebuilt weekly, the ccache mostly are outdated. So the cache is removed from the image. ",
"url": "/docs/v1.5.0/developers/velox-backend-CI.html#ccache",
"relUrl": "/docs/v1.5.0/developers/velox-backend-CI.html#ccache"
- },"865": {
+ },"867": {
"doc": "velox backend CI",
"title": "Updating Docker Image",
"content": "Two GitHub secrets DOCKERHUB_USER & DOCKERHUB_TOKEN can be used to push docker image to docker hub: https://hub.docker.com/r/apache/gluten/tags. Note GitHub secrets are not retrievable in PR from forked repo. ",
"url": "/docs/v1.5.0/developers/velox-backend-CI.html#updating-docker-image",
"relUrl": "/docs/v1.5.0/developers/velox-backend-CI.html#updating-docker-image"
- },"866": {
+ },"868": {
"doc": "velox backend CI",
"title": "velox backend CI",
"content": " ",
"url": "/docs/v1.5.0/developers/velox-backend-CI.html",
"relUrl": "/docs/v1.5.0/developers/velox-backend-CI.html"
- },"867": {
+ },"869": {
"doc": "velox backend build in docker",
"title": "Static link",
"content": "The static link approach builds all dependency libraries in vcpkg for both Velox and Gluten. It then statically links these libraries into libvelox.so and libgluten.so, enabling the build of Gluten on any Linux OS on x86 platforms with 64G memory. However we only verified on Centos-7/8/9 and Ubuntu 20.04/22.04. Please submit an issue if it fails on your OS. Here is the dependency libraries required on target system, they are the essential libraries pre-installed in every [...]
"url": "/docs/v1.5.0/developers/velox-backend-build-in-docker.html#static-link",
"relUrl": "/docs/v1.5.0/developers/velox-backend-build-in-docker.html#static-link"
- },"868": {
+ },"870": {
"doc": "velox backend build in docker",
"title": "Dynamic link",
"content": "The dynamic link approach needs to install the dependencies libraries. It then dynamically link the .so files into libvelox.so and libgluten.so. Currently, Centos-7/8/9 and Ubuntu 20.04/22.04 are supported to build Gluten Velox backend dynamically. The ‘dockerfile’ to build Gluten jar: . FROM apache/gluten:centos-8-jdk8 # Build Gluten Jar RUN source /opt/rh/devtoolset-11/enable && \\ git clone https://github.com/apache/incubator-gluten.git && \\ cd incubat [...]
"url": "/docs/v1.5.0/developers/velox-backend-build-in-docker.html#dynamic-link",
"relUrl": "/docs/v1.5.0/developers/velox-backend-build-in-docker.html#dynamic-link"
- },"869": {
+ },"871": {
"doc": "velox backend build in docker",
"title": "velox backend build in docker",
"content": "Currently, we have two way to build Gluten, static link or dynamic link. ",
"url": "/docs/v1.5.0/developers/velox-backend-build-in-docker.html",
"relUrl": "/docs/v1.5.0/developers/velox-backend-build-in-docker.html"
- },"870": {
+ },"872": {
"doc": "velox function development guide",
"title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
"content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
"url": "/docs/v1.5.0/developers/velox-function-development-guide.html#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
"relUrl": "/docs/v1.5.0/developers/velox-function-development-guide.html#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"871": {
+ },"873": {
"doc": "velox function development guide",
"title": "velox function development guide",
"content": " ",
"url": "/docs/v1.5.0/developers/velox-function-development-guide.html",
"relUrl": "/docs/v1.5.0/developers/velox-function-development-guide.html"
- },"872": {
+ },"874": {
"doc": "ClickHouse",
"title": "ClickHouse Backend",
"content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
"url": "/docs/v1.5.0/get-started/ClickHouse.html#clickhouse-backend",
"relUrl": "/docs/v1.5.0/get-started/ClickHouse.html#clickhouse-backend"
- },"873": {
+ },"875": {
"doc": "ClickHouse",
"title": "ClickHouse",
"content": " ",
"url": "/docs/v1.5.0/get-started/ClickHouse.html",
"relUrl": "/docs/v1.5.0/get-started/ClickHouse.html"
- },"874": {
+ },"876": {
"doc": "GlutenUI",
"title": "Gluten UI",
"content": " ",
"url": "/docs/v1.5.0/get-started/GlutenUI.html#gluten-ui",
"relUrl": "/docs/v1.5.0/get-started/GlutenUI.html#gluten-ui"
- },"875": {
+ },"877": {
"doc": "GlutenUI",
"title": "Gluten event",
"content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
"url": "/docs/v1.5.0/get-started/GlutenUI.html#gluten-event",
"relUrl": "/docs/v1.5.0/get-started/GlutenUI.html#gluten-event"
- },"876": {
+ },"878": {
"doc": "GlutenUI",
"title": "SQL tab",
"content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
"url": "/docs/v1.5.0/get-started/GlutenUI.html#sql-tab",
"relUrl": "/docs/v1.5.0/get-started/GlutenUI.html#sql-tab"
- },"877": {
+ },"879": {
"doc": "GlutenUI",
"title": "History server",
"content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
"url": "/docs/v1.5.0/get-started/GlutenUI.html#history-server",
"relUrl": "/docs/v1.5.0/get-started/GlutenUI.html#history-server"
- },"878": {
+ },"880": {
"doc": "GlutenUI",
"title": "GlutenUI",
"content": " ",
"url": "/docs/v1.5.0/get-started/GlutenUI.html",
"relUrl": "/docs/v1.5.0/get-started/GlutenUI.html"
- },"879": {
+ },"881": {
"doc": "Velox",
"title": "Supported Version",
"content": "| Type | Version | . | Spark | 3.2.2, 3.3.1, 3.4.4, 3.5.5 | . | OS | Ubuntu20.04/22.04, Centos7/8 | . | jdk | openjdk8/jdk17 | . | scala | 2.12 | . ",
"url": "/docs/v1.5.0/get-started/Velox.html#supported-version",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#supported-version"
- },"880": {
+ },"882": {
"doc": "Velox",
"title": "Prerequisite",
"content": "Currently, with static build Gluten+Velox backend supports all the Linux OSes, but is only tested on Ubuntu20.04/Ubuntu22.04/Centos7/Centos8. With dynamic build, Gluten+Velox backend support Ubuntu20.04/Ubuntu22.04/Centos7/Centos8 and their variants. Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.4 and 3.5.5. We need to set up the JAVA_HOME env. Currently, Gluten supports java 8 and java 17. For x86_64 . ## make sure jdk8 is used export JAVA_HOME [...]
"url": "/docs/v1.5.0/get-started/Velox.html#prerequisite",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#prerequisite"
- },"881": {
+ },"883": {
"doc": "Velox",
"title": "Build Gluten with Velox Backend",
"content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . First time build for all supported spark versions./dev/buildbundle-veloxbe.sh . After a complete build, if only some gluten code is changed, you can use the following command to skip building arrow and setting up build dependencies./dev/buildbundle-veloxbe.sh --build_arrow=OFF [...]
"url": "/docs/v1.5.0/get-started/Velox.html#build-gluten-with-velox-backend",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#build-gluten-with-velox-backend"
- },"882": {
+ },"884": {
"doc": "Velox",
"title": "Dependency library deployment",
"content": "With build option enable_vcpkg=ON, all dependency libraries will be statically linked to libvelox.so and libgluten.so which are packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed ( build in docker ). On host with some libraries like jemalloc ins [...]
"url": "/docs/v1.5.0/get-started/Velox.html#dependency-library-deployment",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#dependency-library-deployment"
- },"883": {
+ },"885": {
"doc": "Velox",
"title": "Remote storage support",
"content": " ",
"url": "/docs/v1.5.0/get-started/Velox.html#remote-storage-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#remote-storage-support"
- },"884": {
+ },"886": {
"doc": "Velox",
"title": "HDFS support",
"content": "Gluten supports dynamically loading both libhdfs.so and libhdfs3.so at runtime by using dlopen, allowing the JVM to load the appropriate shared library file as needed. This means you do not need to set the library path during the compilation phase. To enable this functionality, you must set the JAVA_HOME and HADOOP_HOME environment variables. Gluten will then locate and load the ${HADOOP_HOME}/lib/native/libhdfs.so file at runtime. If you prefer to use libhdfs3.so instead [...]
"url": "/docs/v1.5.0/get-started/Velox.html#hdfs-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#hdfs-support"
- },"885": {
+ },"887": {
"doc": "Velox",
"title": "Azure Blob File System (ABFS) support",
"content": "Velox supports ABFS with the open source Azure SDK for C++ and Gluten uses the Velox ABFS connector to connect with ABFS. The build option for ABFS (enable_abfs) must be set to enable this feature as listed below. cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_abfs=ON . Please refer Velox ABFS part for more detailed configurations. ",
"url": "/docs/v1.5.0/get-started/Velox.html#azure-blob-file-system-abfs-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#azure-blob-file-system-abfs-support"
- },"886": {
+ },"888": {
"doc": "Velox",
"title": "AWS S3 support",
"content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to access S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
"url": "/docs/v1.5.0/get-started/Velox.html#aws-s3-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#aws-s3-support"
- },"887": {
+ },"889": {
"doc": "Velox",
"title": "GCS support",
"content": "Please refer GCS . ",
"url": "/docs/v1.5.0/get-started/Velox.html#gcs-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#gcs-support"
- },"888": {
+ },"890": {
"doc": "Velox",
"title": "Remote Shuffle Service Support",
"content": " ",
"url": "/docs/v1.5.0/get-started/Velox.html#remote-shuffle-service-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#remote-shuffle-service-support"
- },"889": {
+ },"891": {
"doc": "Velox",
"title": "Celeborn support",
"content": "Gluten with velox backend supports Celeborn as remote shuffle service. Currently, the supported Celeborn versions are 0.3.x, 0.4.x and 0.5.x. Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable celeborn profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests . Then add the Gluten and Spa [...]
"url": "/docs/v1.5.0/get-started/Velox.html#celeborn-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#celeborn-support"
- },"890": {
+ },"892": {
"doc": "Velox",
"title": "Uniffle support",
"content": "Uniffle with velox backend supports Uniffle as remote shuffle service. Currently, the supported Uniffle versions are 0.9.2. First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with uniffle. When compiling the Gluten Java module, it’s required to enable uniffle profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests . Then add the Uniffle and Spark Celeborn Client packages to your Spark application’s classpath (us [...]
"url": "/docs/v1.5.0/get-started/Velox.html#uniffle-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#uniffle-support"
- },"891": {
+ },"893": {
"doc": "Velox",
"title": "Datalake Framework Support",
"content": " ",
"url": "/docs/v1.5.0/get-started/Velox.html#datalake-framework-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#datalake-framework-support"
- },"892": {
+ },"894": {
"doc": "Velox",
"title": "DeltaLake Support",
"content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Once built successfully, delta features will be included in gluten-velox-bundle-X jar. Then you can query delta table by gluten/velox without scan’s fallback. Gluten with velox backends also support the column mapping of delta tables. About column mapping, see more [...]
"url": "/docs/v1.5.0/get-started/Velox.html#deltalake-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#deltalake-support"
- },"893": {
+ },"895": {
"doc": "Velox",
"title": "Iceberg Support",
"content": "Gluten with velox backend supports Iceberg table. Currently, both reading COW (Copy-On-Write) and MOR (Merge-On-Read) tables are supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan’s fallback. ",
"url": "/docs/v1.5.0/get-started/Velox.html#iceberg-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#iceberg-support"
- },"894": {
+ },"896": {
"doc": "Velox",
"title": "Hudi Support",
"content": "Gluten with velox backend supports Hudi table. Currently, only reading COW (Copy-On-Write) tables is supported. ",
"url": "/docs/v1.5.0/get-started/Velox.html#hudi-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#hudi-support"
- },"895": {
+ },"897": {
"doc": "Velox",
"title": "Paimon Support",
"content": "Gluten with velox backend supports Paimon table. Currently, only non-pk table is supported, and the Spark version needs to be >= 3.3. How to use . Compile gluten-paimon module by a paimon profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.5 -Ppaimon -DskipTests . Once built successfully, paimon features will be included in gluten-velox-bundle-X jar. Then you can query paimon non-pk table by gluten/velox without scan’s fallback. How to use . First of al [...]
"url": "/docs/v1.5.0/get-started/Velox.html#paimon-support",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#paimon-support"
- },"896": {
+ },"898": {
"doc": "Velox",
"title": "Coverage",
"content": "Spark3.3 has 387 functions in total. ~240 are commonly used. To get the support status of all Spark built-in functions, please refer to Velox Backend’s Supported Operators & Functions. Velox doesn’t support ANSI mode), so as Gluten. Once ANSI mode is enabled in Spark config, Gluten will fallback to Vanilla Spark. To identify what can be offloaded in a query and detailed fallback reasons, user can follow below steps to retrieve corresponding logs. 1) Enable Gluten by p [...]
"url": "/docs/v1.5.0/get-started/Velox.html#coverage",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#coverage"
- },"897": {
+ },"899": {
"doc": "Velox",
"title": "Spill",
"content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
"url": "/docs/v1.5.0/get-started/Velox.html#spill",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#spill"
- },"898": {
+ },"900": {
"doc": "Velox",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": "Please check the VeloxNativeUDF.md for more detailed usage and configurations. ",
"url": "/docs/v1.5.0/get-started/Velox.html#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"899": {
+ },"901": {
"doc": "Velox",
"title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
"content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. You may refer to the notebook we used to do the performance test. ",
"url": "/docs/v1.5.0/get-started/Velox.html#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
- },"900": {
+ },"902": {
"doc": "Velox",
"title": "Data preparation",
"content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. ",
"url": "/docs/v1.5.0/get-started/Velox.html#data-preparation",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#data-preparation"
- },"901": {
+ },"903": {
"doc": "Velox",
"title": "Submit the Spark SQL job",
"content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
"url": "/docs/v1.5.0/get-started/Velox.html#submit-the-spark-sql-job",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#submit-the-spark-sql-job"
- },"902": {
+ },"904": {
"doc": "Velox",
"title": "Result",
"content": "wholestagetransformer indicates that the offloading works. ",
"url": "/docs/v1.5.0/get-started/Velox.html#result",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#result"
- },"903": {
+ },"905": {
"doc": "Velox",
"title": "Performance",
"content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
"url": "/docs/v1.5.0/get-started/Velox.html#performance",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#performance"
- },"904": {
+ },"906": {
"doc": "Velox",
"title": "Gluten UI",
"content": "Please refer Gluten UI . ",
"url": "/docs/v1.5.0/get-started/Velox.html#gluten-ui",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#gluten-ui"
- },"905": {
+ },"907": {
"doc": "Velox",
"title": "Gluten Native Plan Summary",
"content": " ",
"url": "/docs/v1.5.0/get-started/Velox.html#gluten-native-plan-summary",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#gluten-native-plan-summary"
- },"906": {
+ },"908": {
"doc": "Velox",
"title": "Gluten Implicits",
"content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
"url": "/docs/v1.5.0/get-started/Velox.html#gluten-implicits",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#gluten-implicits"
- },"907": {
+ },"909": {
"doc": "Velox",
"title": "Native Plan in Spark’s Explain Output",
"content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten shows the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \ [...]
"url": "/docs/v1.5.0/get-started/Velox.html#native-plan-in-sparks-explain-output",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#native-plan-in-sparks-explain-output"
- },"908": {
+ },"910": {
"doc": "Velox",
"title": "Native Plan with Stats",
"content": "Gluten supports print native plan with statistics to executor system output stream by setting --conf spark.gluten.sql.columnar.backend.velox.showTaskMetricsWhenFinished=true or --conf spark.gluten.sql.debug=true. Note that the plan string with statistics is task level, which may increase the size of the executor logs. Below is an example of how Gluten displays the native plan string with statistics. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Nativ [...]
"url": "/docs/v1.5.0/get-started/Velox.html#native-plan-with-stats",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#native-plan-with-stats"
- },"909": {
+ },"911": {
"doc": "Velox",
"title": "Using Stage-Level Resource Adjustment to Avoid OOM(Experimental)",
"content": "see more here . ",
"url": "/docs/v1.5.0/get-started/Velox.html#using-stage-level-resource-adjustment-to-avoid-oomexperimental",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#using-stage-level-resource-adjustment-to-avoid-oomexperimental"
- },"910": {
+ },"912": {
"doc": "Velox",
"title": "Broadcast Build Relations to Off-Heap(Experimental)",
"content": "The experimental feature Off-Heap Broadcast Build Relations aims to mitigate out-of-memory (OOM) issues caused by heap memory consumption during broadcast operations. Detailed design can be found here . Purpose & how it works . | Avoid OOM: Prevent OOM errors when broadcasting large datasets. | Reduce Heap Memory Usage: Store broadcast build relations in Spark off-heap memory instead of on-heap memory | . Configuration . Enable Off-Heap Broadcast . To enable this feat [...]
"url": "/docs/v1.5.0/get-started/Velox.html#broadcast-build-relations-to-off-heapexperimental",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#broadcast-build-relations-to-off-heapexperimental"
- },"911": {
+ },"913": {
"doc": "Velox",
"title": "Accelerators",
"content": "Please refer QAT for details . ",
"url": "/docs/v1.5.0/get-started/Velox.html#accelerators",
"relUrl": "/docs/v1.5.0/get-started/Velox.html#accelerators"
- },"912": {
+ },"914": {
"doc": "Velox",
"title": "Velox",
"content": " ",
"url": "/docs/v1.5.0/get-started/Velox.html",
"relUrl": "/docs/v1.5.0/get-started/Velox.html"
- },"913": {
+ },"915": {
"doc": "VeloxABFS",
"title": "Working with ABFS",
"content": " ",
"url": "/docs/v1.5.0/get-started/VeloxABFS.html#working-with-abfs",
"relUrl": "/docs/v1.5.0/get-started/VeloxABFS.html#working-with-abfs"
- },"914": {
+ },"916": {
"doc": "VeloxABFS",
"title": "Configuring ABFS Access Token",
"content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
"url": "/docs/v1.5.0/get-started/VeloxABFS.html#configuring-abfs-access-token",
"relUrl": "/docs/v1.5.0/get-started/VeloxABFS.html#configuring-abfs-access-token"
- },"915": {
+ },"917": {
"doc": "VeloxABFS",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/docs/v1.5.0/get-started/VeloxABFS.html#local-caching-support",
"relUrl": "/docs/v1.5.0/get-started/VeloxABFS.html#local-caching-support"
- },"916": {
+ },"918": {
"doc": "VeloxABFS",
"title": "VeloxABFS",
"content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
"url": "/docs/v1.5.0/get-started/VeloxABFS.html",
"relUrl": "/docs/v1.5.0/get-started/VeloxABFS.html"
- },"917": {
+ },"919": {
"doc": "VeloxGCS",
"title": "Working with GCS",
"content": " ",
"url": "/docs/v1.5.0/get-started/VeloxGCS.html#working-with-gcs",
"relUrl": "/docs/v1.5.0/get-started/VeloxGCS.html#working-with-gcs"
- },"918": {
+ },"920": {
"doc": "VeloxGCS",
"title": "Installing the gcloud CLI",
"content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
"url": "/docs/v1.5.0/get-started/VeloxGCS.html#installing-the-gcloud-cli",
"relUrl": "/docs/v1.5.0/get-started/VeloxGCS.html#installing-the-gcloud-cli"
- },"919": {
+ },"921": {
"doc": "VeloxGCS",
"title": "Configuring GCS using a user account",
"content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
"url": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-using-a-user-account",
"relUrl": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-using-a-user-account"
- },"920": {
+ },"922": {
"doc": "VeloxGCS",
"title": "Configuring GCS using a credential file",
"content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credentials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to t [...]
"url": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-using-a-credential-file",
"relUrl": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-using-a-credential-file"
- },"921": {
+ },"923": {
"doc": "VeloxGCS",
"title": "Configuring GCS endpoints",
"content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
"url": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-endpoints",
"relUrl": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-endpoints"
- },"922": {
+ },"924": {
"doc": "VeloxGCS",
"title": "Configuring GCS max retry count",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
"url": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-max-retry-count",
"relUrl": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-max-retry-count"
- },"923": {
+ },"925": {
"doc": "VeloxGCS",
"title": "Configuring GCS max retry time",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retrying (10s, 1m, etc). ",
"url": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-max-retry-time",
"relUrl": "/docs/v1.5.0/get-started/VeloxGCS.html#configuring-gcs-max-retry-time"
- },"924": {
+ },"926": {
"doc": "VeloxGCS",
"title": "VeloxGCS",
"content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/docs/v1.5.0/get-started/VeloxGCS.html",
"relUrl": "/docs/v1.5.0/get-started/VeloxGCS.html"
- },"925": {
+ },"927": {
"doc": "VeloxGPU",
"title": "GPU Acceleration in Velox/Gluten",
"content": "Unified execution engine leveraging CUDF for hardware-accelerated Spark SQL queries . ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html#gpu-acceleration-in-veloxgluten",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html#gpu-acceleration-in-veloxgluten"
- },"926": {
+ },"928": {
"doc": "VeloxGPU",
"title": "1. Overview",
"content": ". | Purpose: Accelerate Velox operators via CUDF APIs, replacing CPU execution when enabled. | Status: Experimental (TPC-H SF1 validated). Integrates RAPIDS ecosystem with Apache Spark via Gluten . | Key Benefit: Some queries achieved up to 8.1x speedup on x86 vs. Spark Java engine . | . ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html#1-overview",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html#1-overview"
- },"927": {
+ },"929": {
"doc": "VeloxGPU",
"title": "2. Prerequisites",
"content": ". | CUDA Toolkit: 12.8.0 (download). | NVIDIA Drivers: Compatible with CUDA 12.8. | Container Toolkit: Install nvidia-container-toolkit (guide). | System Reboot: Required after driver installation. | Environment Setup: Use start_cudf.sh for host configuration . | . ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html#2-prerequisites",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html#2-prerequisites"
- },"928": {
+ },"930": {
"doc": "VeloxGPU",
"title": "3. Implementation Mechanics",
"content": ". | Operator Conversion: . | Velox PlanNodes → GPU operators when spark.gluten.sql.columnar.cudf=true. | Falls back to CPU operators if GPU unsupported (triggers row/columnar data conversion) . | . | Debugging: Enable spark.gluten.debug.enabled.cudf=true for operator replacement logs. | Memory: Global RMM memory manager, cannot align with Spark memory system. | . ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html#3-implementation-mechanics",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html#3-implementation-mechanics"
- },"929": {
+ },"931": {
"doc": "VeloxGPU",
"title": "4. Docker Deployment",
"content": "docker pull apache/gluten:centos-9-jdk8-cudf # Pre-built GPU image docker run --name gpu_gluten_container --gpus all -it apache/gluten:centos-9-jdk8-cudf . | Image Includes: Native build cache, Gluten dependencies, Spark 3.4 environment. | . ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html#4-docker-deployment",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html#4-docker-deployment"
- },"930": {
+ },"932": {
"doc": "VeloxGPU",
"title": "5. Build & Deployment",
"content": "Dependencies . The OS, Spark version, Java version aligns with Gluten CPU. Compilation Commands . If building in the docker image, no need to set up script and build arrow./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --enable_cudf=ON . ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html#5-build--deployment",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html#5-build--deployment"
- },"931": {
+ },"933": {
"doc": "VeloxGPU",
"title": "6. GPU Operator Support Status",
"content": "| Operator | Status | Notes |—————–|—————–|————————–| Scan | ❌ Not supported| In Development | Project | ⚠️ Partial | Function TPCH-compatible | Filter | ✅ Implemented | Core operator | OrderBy | ✅ Implemented | Merged in Velox #12735 | Aggregation | ⚠️ Partial | TPCH-compatible | Join | ⚠️ Partial | TPCH-compatible | Spill | ❌ Not supported | In Planning | . ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html#6-gpu-operator-support-status",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html#6-gpu-operator-support-status"
- },"932": {
+ },"934": {
"doc": "VeloxGPU",
"title": "7. Performance Validation",
"content": "GPU performs better on operator HashJoin and HashAggregation. Single Operator like Hash Agg shows 5x speedup. ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html#7-performance-validation",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html#7-performance-validation"
- },"933": {
+ },"935": {
"doc": "VeloxGPU",
"title": "8. Relevant Resources",
"content": ". | CUDF Docs - GPU operator APIs. | Gluten GPU Issue #9098 - Development tracker. | . ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html#8-relevant-resources",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html#8-relevant-resources"
- },"934": {
+ },"936": {
"doc": "VeloxGPU",
"title": "VeloxGPU",
"content": " ",
"url": "/docs/v1.5.0/get-started/VeloxGPU.html",
"relUrl": "/docs/v1.5.0/get-started/VeloxGPU.html"
- },"935": {
+ },"937": {
"doc": "VeloxIceberg",
"title": "Iceberg Support in Velox Backend",
"content": " ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#iceberg-support-in-velox-backend",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#iceberg-support-in-velox-backend"
- },"936": {
+ },"938": {
"doc": "VeloxIceberg",
"title": "Supported Spark version",
"content": "All the spark version is supported, but for convenience, only Spark 3.4 is well tested. Now only read is supported in Gluten. ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#supported-spark-version",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#supported-spark-version"
- },"937": {
+ },"939": {
"doc": "VeloxIceberg",
"title": "Support Status",
"content": "Following value indicates the iceberg support progress: . | Value | Description | . | Offload | Offload to the Velox backend | . | PartialOffload | Some operators offload and some fallback | . | Fallback | Fallback to spark to execute | . | Exception | Cannot fallback by some conditions, throw the exception | . | ResultMismatch | Some hidden bug may cause result mismatch, especially for some corner case | . ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#support-status",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#support-status"
- },"938": {
+ },"940": {
"doc": "VeloxIceberg",
"title": "Adding catalogs",
"content": "Fallback . ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#adding-catalogs",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#adding-catalogs"
- },"939": {
+ },"941": {
"doc": "VeloxIceberg",
"title": "Creating a table",
"content": "Fallback . ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#creating-a-table",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#creating-a-table"
- },"940": {
+ },"942": {
"doc": "VeloxIceberg",
"title": "Writing",
"content": "Fallback . INSERT INTO local.db.table VALUES (1, 'a'), (2, 'b'), (3, 'c'); . PartialOffload . The write is fallback while read is offload. INSERT INTO local.db.table SELECT id, data FROM source WHERE length(data) = 1; . ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#writing",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#writing"
- },"941": {
+ },"943": {
"doc": "VeloxIceberg",
"title": "Reading",
"content": "Read data . Offload/Fallback . | Table Type | No Delete | Position Delete | Equality Delete | . | unpartition | Offload | Offload | Fallback | . | partition | Fallback mostly | Fallback mostly | Fallback | . | metadata | Fallback | | | . Offload the simple query. SELECT count(1) as count, data FROM local.db.table GROUP BY data; . If delete by Spark and copy on read, will generate position delete file, the query may offload. If delete by Flink, may generate the equalit [...]
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#reading",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#reading"
- },"942": {
+ },"944": {
"doc": "VeloxIceberg",
"title": "DataType",
"content": "Timestamptz in orc format is not supported, throws exception. UUID type and Fixed type is fallback. ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#datatype",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#datatype"
- },"943": {
+ },"945": {
"doc": "VeloxIceberg",
"title": "Format",
"content": "PartialOffload . Supports parquet and orc format. Not support avro format. ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#format",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#format"
- },"944": {
+ },"946": {
"doc": "VeloxIceberg",
"title": "SQL",
"content": "Only support SELECT. ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#sql",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#sql"
- },"945": {
+ },"947": {
"doc": "VeloxIceberg",
"title": "Schema evolution",
"content": "PartialOffload . Gluten uses column name to match the parquet file, so if the column is renamed or the added column name is same to the deleted column, the scan will fall back. ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#schema-evolution",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#schema-evolution"
- },"946": {
+ },"948": {
"doc": "VeloxIceberg",
"title": "Configuration",
"content": "Catalogs . Supports all the catalog options, which is not used in native engine. SQL Extensions . Fallback . Supports the option spark.sql.extensions, fallback the SQL command CALL. Runtime configuration . Read options . | Spark option | Status | . | snapshot-id | Support | . | as-of-timestamp | Support | . | split-size | Support | . | lookback | Support | . | file-open-cost | Support | . | vectorization-enabled | Not Support | . | batch-size | Not Support | . ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html#configuration",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html#configuration"
- },"947": {
+ },"949": {
"doc": "VeloxIceberg",
"title": "VeloxIceberg",
"content": " ",
"url": "/docs/v1.5.0/get-started/VeloxIceberg.html",
"relUrl": "/docs/v1.5.0/get-started/VeloxIceberg.html"
- },"948": {
+ },"950": {
"doc": "VeloxLocalCache",
"title": "VeloxLocalCache",
"content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
"url": "/docs/v1.5.0/get-started/VeloxLocalCache.html",
"relUrl": "/docs/v1.5.0/get-started/VeloxLocalCache.html"
- },"949": {
+ },"951": {
"doc": "VeloxQAT",
"title": "Intel® QuickAssist Technology (QAT) support",
"content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
"url": "/docs/v1.5.0/get-started/VeloxQAT.html#intel-quickassist-technology-qat-support",
"relUrl": "/docs/v1.5.0/get-started/VeloxQAT.html#intel-quickassist-technology-qat-support"
- },"950": {
+ },"952": {
"doc": "VeloxQAT",
"title": "Software Requirements",
"content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
"url": "/docs/v1.5.0/get-started/VeloxQAT.html#software-requirements",
"relUrl": "/docs/v1.5.0/get-started/VeloxQAT.html#software-requirements"
- },"951": {
+ },"953": {
"doc": "VeloxQAT",
"title": "Build Gluten with QAT",
"content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
"url": "/docs/v1.5.0/get-started/VeloxQAT.html#build-gluten-with-qat",
"relUrl": "/docs/v1.5.0/get-started/VeloxQAT.html#build-gluten-with-qat"
- },"952": {
+ },"954": {
"doc": "VeloxQAT",
"title": "Enable QAT with Gzip/Zstd for shuffle compression",
"content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
"url": "/docs/v1.5.0/get-started/VeloxQAT.html#enable-qat-with-gzipzstd-for-shuffle-compression",
"relUrl": "/docs/v1.5.0/get-started/VeloxQAT.html#enable-qat-with-gzipzstd-for-shuffle-compression"
- },"953": {
+ },"955": {
"doc": "VeloxQAT",
"title": "QAT driver references",
"content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
"url": "/docs/v1.5.0/get-started/VeloxQAT.html#qat-driver-references",
"relUrl": "/docs/v1.5.0/get-started/VeloxQAT.html#qat-driver-references"
- },"954": {
+ },"956": {
"doc": "VeloxQAT",
"title": "VeloxQAT",
"content": " ",
"url": "/docs/v1.5.0/get-started/VeloxQAT.html",
"relUrl": "/docs/v1.5.0/get-started/VeloxQAT.html"
- },"955": {
+ },"957": {
"doc": "VeloxS3",
"title": "Working with S3",
"content": " ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html#working-with-s3",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#working-with-s3"
- },"956": {
+ },"958": {
"doc": "VeloxS3",
"title": "Configuring S3 endpoint",
"content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
"url": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-endpoint",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-endpoint"
- },"957": {
+ },"959": {
"doc": "VeloxS3",
"title": "Configuring S3 instance credentials",
"content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-instance-credentials",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-instance-credentials"
- },"958": {
+ },"960": {
"doc": "VeloxS3",
"title": "Configuring S3 IAM roles",
"content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-iam-roles",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-iam-roles"
- },"959": {
+ },"961": {
"doc": "VeloxS3",
"title": "Other authentatication methods are not supported yet",
"content": ". | AWS temporary credential | . ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html#other-authentatication-methods-are-not-supported-yet",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#other-authentatication-methods-are-not-supported-yet"
- },"960": {
+ },"962": {
"doc": "VeloxS3",
"title": "Log granularity of AWS C++ SDK in velox",
"content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: “OFF”, “FATAL”, “ERROR”, “WARN”, “INFO”, “DEBUG”, “TRACE”. ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html#log-granularity-of-aws-c-sdk-in-velox",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#log-granularity-of-aws-c-sdk-in-velox"
- },"961": {
+ },"963": {
"doc": "VeloxS3",
"title": "Configuring Whether To Use Proxy From Env for S3 C++ Client",
"content": "You can change whether to use proxy from env for S3 C++ client by setting the spark.gluten.velox.s3UseProxyFromEnv configuration. The Allowed values are: “false”, “true”. ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-whether-to-use-proxy-from-env-for-s3-c-client",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-whether-to-use-proxy-from-env-for-s3-c-client"
- },"962": {
+ },"964": {
"doc": "VeloxS3",
"title": "Configuring S3 Payload Signing Policy",
"content": "You can change the S3 payload signing policy by setting the spark.gluten.velox.s3PayloadSigningPolicy configuration. The Allowed values are: “Always”, “RequestDependent”, “Never”. | When set to “Always”, the payload checksum is included in the signature calculation. | When set to “RequestDependent”, the payload checksum is included based on the value returned by “AmazonWebServiceRequest::SignBody()”. | . ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-payload-signing-policy",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-payload-signing-policy"
- },"963": {
+ },"965": {
"doc": "VeloxS3",
"title": "Configuring S3 Log Location",
"content": "You can set the log location by setting the spark.gluten.velox.s3LogLocation configuration. ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-log-location",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#configuring-s3-log-location"
- },"964": {
+ },"966": {
"doc": "VeloxS3",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html#local-caching-support",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html#local-caching-support"
- },"965": {
+ },"967": {
"doc": "VeloxS3",
"title": "VeloxS3",
"content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/docs/v1.5.0/get-started/VeloxS3.html",
"relUrl": "/docs/v1.5.0/get-started/VeloxS3.html"
- },"966": {
+ },"968": {
"doc": "VeloxStageResourceAdj",
"title": "Using Stage-Level Resource Adjustment to Avoid OOM(Experimental)",
"content": "Overview . One major advantage of Apache Gluten is its ability to significantly reduce memory requirements per executor—potentially by up to half—when entire stages are offloaded to the native engine. This engine primarily relies on off-heap memory with minimal on-heap usage. However, when stages contain fallback operators that utilize the JVM engine, the on-heap memory size must be increased, leading to even higher memory demands per executor. This challenge has posed si [...]
"url": "/docs/v1.5.0/get-started/VeloxStageResourceAdj.html#using-stage-level-resource-adjustment-to-avoid-oomexperimental",
"relUrl": "/docs/v1.5.0/get-started/VeloxStageResourceAdj.html#using-stage-level-resource-adjustment-to-avoid-oomexperimental"
- },"967": {
+ },"969": {
"doc": "VeloxStageResourceAdj",
"title": "VeloxStageResourceAdj",
"content": " ",
"url": "/docs/v1.5.0/get-started/VeloxStageResourceAdj.html",
"relUrl": "/docs/v1.5.0/get-started/VeloxStageResourceAdj.html"
- },"968": {
+ },"970": {
"doc": "build guide",
"title": "Build Parameters",
"content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | enable_jemalloc_stats | Print jemalloc stats for debugging. | [...]
"url": "/docs/v1.5.0/get-started/build-guide.html#build-parameters",
"relUrl": "/docs/v1.5.0/get-started/build-guide.html#build-parameters"
- },"969": {
+ },"971": {
"doc": "build guide",
"title": "Gluten Jar for Deployment",
"content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.4 | 3.4 | 2.12 | . | 3.5.5 | 3.5 | 2.12 | . ",
"url": "/docs/v1.5.0/get-started/build-guide.html#gluten-jar-for-deployment",
"relUrl": "/docs/v1.5.0/get-started/build-guide.html#gluten-jar-for-deployment"
- },"970": {
+ },"972": {
"doc": "build guide",
"title": "build guide",
"content": " ",
"url": "/docs/v1.5.0/get-started/build-guide.html",
"relUrl": "/docs/v1.5.0/get-started/build-guide.html"
- },"971": {
+ },"973": {
"doc": "Getting-Started",
"title": "Getting Started with Gluten for Apache Spark",
"content": " ",
"url": "/archives/v1.5.0/getting-started#getting-started-with-gluten-for-apache-spark",
"relUrl": "/archives/v1.5.0/getting-started#getting-started-with-gluten-for-apache-spark"
- },"972": {
+ },"974": {
"doc": "Getting-Started",
"title": "Getting-Started",
"content": " ",
"url": "/archives/v1.5.0/getting-started",
"relUrl": "/archives/v1.5.0/getting-started"
- },"973": {
+ },"975": {
"doc": "v1.5.0",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.5.0/#gluten-documents-by-version",
"relUrl": "/archives/v1.5.0/#gluten-documents-by-version"
- },"974": {
+ },"976": {
"doc": "v1.5.0",
"title": "v1.5.0",
"content": " ",
"url": "/archives/v1.5.0/",
"relUrl": "/archives/v1.5.0/"
- },"975": {
+ },"977": {
"doc": "Velox Backend",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.5.0/velox-backend#gluten-documents-by-version",
"relUrl": "/archives/v1.5.0/velox-backend#gluten-documents-by-version"
- },"976": {
+ },"978": {
"doc": "Velox Backend",
"title": "Velox Backend",
"content": " ",
"url": "/archives/v1.5.0/velox-backend",
"relUrl": "/archives/v1.5.0/velox-backend"
- },"977": {
+ },"979": {
"doc": "velox backend aggregate function support",
"title": "Aggregate Functions Support Status",
"content": "Out of 62 aggregate functions in Spark 3.5, Gluten currently fully supports 54 functions and partially supports 1 function. ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html#aggregate-functions-support-status",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html#aggregate-functions-support-status"
- },"978": {
+ },"980": {
"doc": "velox backend aggregate function support",
"title": "Aggregate Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | any | BoolOr | S | | . | any_value | AnyValue | S | | . | approx_count_distinct | HyperLogLogPlusPlus | S | | . | approx_percentile | ApproximatePercentile | S | | . | array_agg | CollectList | S | | . | avg | Average | S | | . | bit_and | BitAndAgg | S | | . | bit_or | BitOrAgg | S | | . | bit_xor | BitXorAgg | S | | . | bitmap_construct_agg | BitmapConstructAgg | | | . | bitmap_or_ [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html#aggregate-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html#aggregate-functions"
- },"979": {
+ },"981": {
"doc": "velox backend aggregate function support",
"title": "velox backend aggregate function support",
"content": " ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html"
- },"980": {
+ },"982": {
"doc": "velox backend generator function support",
"title": "Generator Functions Support Status",
"content": "Out of 7 generator functions in Spark 3.5, Gluten currently fully supports 7 functions. ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html#generator-functions-support-status",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html#generator-functions-support-status"
- },"981": {
+ },"983": {
"doc": "velox backend generator function support",
"title": "Generator Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | explode | ExplodeExpressionBuilder | S | | . | explode_outer | ExplodeExpressionBuilder | S | | . | inline | Inline | S | | . | inline_outer | Inline | S | | . | posexplode | PosExplode | S | | . | posexplode_outer | PosExplode | S | | . | stack | Stack | S | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html#generator-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html#generator-functions"
- },"982": {
+ },"984": {
"doc": "velox backend generator function support",
"title": "velox backend generator function support",
"content": " ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html"
- },"983": {
+ },"985": {
"doc": "velox backend limitations",
"title": "velox backend limitations",
"content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-limitations.html",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-limitations.html"
- },"984": {
+ },"986": {
"doc": "velox backend scalar function support",
"title": "Scalar Functions Support Status",
"content": "Out of 357 scalar functions in Spark 3.5, Gluten currently fully supports 239 functions and partially supports 24 functions. ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#scalar-functions-support-status",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#scalar-functions-support-status"
- },"985": {
+ },"987": {
"doc": "velox backend scalar function support",
"title": "Array Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | array | CreateArray | S | | . | array_append | ArrayAppend | S | | . | array_compact | ArrayCompact | S | | . | array_contains | ArrayContains | S | | . | array_distinct | ArrayDistinct | S | | . | array_except | ArrayExcept | S | | . | array_insert | ArrayInsert | S | | . | array_intersect | ArrayIntersect | S | | . | array_join | ArrayJoin | S | | . | array_max | ArrayMax | S | | . | [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#array-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#array-functions"
- },"986": {
+ },"988": {
"doc": "velox backend scalar function support",
"title": "Bitwise Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | & | BitwiseAnd | S | | . | ^ | BitwiseXor | S | | . | bit_count | BitwiseCount | S | | . | bit_get | BitwiseGet | S | | . | getbit | BitwiseGet | S | | . | shiftright | ShiftRight | S | | . | shiftrightunsigned | ShiftRightUnsigned | | | . | | BitwiseOr | S | | . | ~ | BitwiseNot | S | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#bitwise-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#bitwise-functions"
- },"987": {
+ },"989": {
"doc": "velox backend scalar function support",
"title": "Collection Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | array_size | ArraySize | S | | . | cardinality | Size | S | | . | concat | Concat | S | | . | reverse | Reverse | S | | . | size | Size | S | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#collection-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#collection-functions"
- },"988": {
+ },"990": {
"doc": "velox backend scalar function support",
"title": "Conditional Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | coalesce | Coalesce | S | | . | if | If | S | | . | ifnull | Nvl | S | | . | nanvl | NaNvl | S | | . | nullif | NullIf | S | | . | nvl | Nvl | S | | . | nvl2 | Nvl2 | S | | . | when | CaseWhen | S | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#conditional-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#conditional-functions"
- },"989": {
+ },"991": {
"doc": "velox backend scalar function support",
"title": "Conversion Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | bigint | | S | | . | binary | | S | | . | boolean | | S | | . | cast | Cast | S | | . | date | | S | | . | decimal | | S | | . | double | | S | | . | float | | S | | . | int | | S | | . | smallint | | S | | . | string | | S | | . | timestamp | | S | | . | tinyint | | S | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#conversion-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#conversion-functions"
- },"990": {
+ },"992": {
"doc": "velox backend scalar function support",
"title": "Csv Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | from_csv | CsvToStructs | | | . | schema_of_csv | SchemaOfCsv | | | . | to_csv | StructsToCsv | | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#csv-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#csv-functions"
- },"991": {
+ },"993": {
"doc": "velox backend scalar function support",
"title": "Date and Timestamp Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | add_months | AddMonths | S | | . | convert_timezone | ConvertTimezone | | | . | curdate | CurDateExpressionBuilder | | | . | current_date | CurrentDate | | | . | current_timestamp | CurrentTimestamp | | | . | current_timezone | CurrentTimeZone | | | . | date_add | DateAdd | S | | . | date_diff | DateDiff | S | | . | date_format | DateFormatClass | S | | . | date_from_unix_date [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#date-and-timestamp-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#date-and-timestamp-functions"
- },"992": {
+ },"994": {
"doc": "velox backend scalar function support",
"title": "Hash Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | crc32 | Crc32 | S | | . | hash | Murmur3Hash | S | | . | md5 | Md5 | S | | . | sha | Sha1 | S | | . | sha1 | Sha1 | S | | . | sha2 | Sha2 | S | | . | xxhash64 | XxHash64 | S | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#hash-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#hash-functions"
- },"993": {
+ },"995": {
"doc": "velox backend scalar function support",
"title": "JSON Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | from_json | JsonToStructs | PS | from_json with ‘spark.sql.caseSensitive = true’ is not supported in Veloxfrom_json with ‘spark.sql.json.enablePartialResults = false’ is not supported in Veloxfrom_json with column corrupt record is not supported in Veloxfrom_json with duplicate keys is not supported in Veloxfrom_json with options is not supported in Velox | . | get_json_object | GetJsonObject | S | | . [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#json-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#json-functions"
- },"994": {
+ },"996": {
"doc": "velox backend scalar function support",
"title": "Lambda Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | aggregate | ArrayAggregate | S | | . | array_sort | ArraySort | S | | . | exists | ArrayExists | S | | . | filter | ArrayFilter | S | | . | forall | ArrayForAll | S | | . | map_filter | MapFilter | S | | . | map_zip_with | MapZipWith | S | | . | reduce | ArrayAggregate | S | | . | transform | ArrayTransform | S | | . | transform_keys | TransformKeys | S | | . | transform_values | Trans [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#lambda-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#lambda-functions"
- },"995": {
+ },"997": {
"doc": "velox backend scalar function support",
"title": "Map Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | element_at | ElementAt | S | | . | map | CreateMap | PS | | . | map_concat | MapConcat | PS | | . | map_contains_key | MapContainsKey | S | | . | map_entries | MapEntries | S | | . | map_from_arrays | MapFromArrays | | | . | map_from_entries | MapFromEntries | | | . | map_keys | MapKeys | S | | . | map_values | MapValues | S | | . | str_to_map | StringToMap | PS | Only spark.sql.mapK [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#map-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#map-functions"
- },"996": {
+ },"998": {
"doc": "velox backend scalar function support",
"title": "Mathematical Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | % | Remainder | S | | . | * | Multiply | S | | . | + | Add | S | | . | - | Subtract | S | | . | / | Divide | S | | . | abs | Abs | S | | . | acos | Acos | S | | . | acosh | Acosh | S | | . | asin | Asin | S | | . | asinh | Asinh | S | | . | atan | Atan | S | | . | atan2 | Atan2 | S | | . | atanh | Atanh | S | | . | bin | Bin | S | | . | bround | BRound | | | . | cbrt | Cbrt [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#mathematical-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#mathematical-functions"
- },"997": {
+ },"999": {
"doc": "velox backend scalar function support",
"title": "Misc Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | aes_decrypt | AesDecrypt | | | . | aes_encrypt | AesEncrypt | | | . | assert_true | AssertTrue | S | | . | bitmap_bit_position | BitmapBitPosition | | | . | bitmap_bucket_number | BitmapBucketNumber | | | . | bitmap_count | BitmapCount | | | . | current_catalog | CurrentCatalog | | | . | current_database | CurrentDatabase | | | . | current_schema | CurrentDatabase | | | . [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#misc-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#misc-functions"
- },"998": {
+ },"1000": {
"doc": "velox backend scalar function support",
"title": "Predicate Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | ! | Not | S | | . | != | | S | | . | < | LessThan | S | | . | <= | LessThanOrEqual | S | | . | <=> | EqualNullSafe | S | | . | <> | | S | | . | = | EqualTo | S | | . | == | EqualTo | S | | . | > | GreaterThan | S | | . | >= | GreaterThanOrEqual | S | | . | and | And | S | | . | between | | S | | . | case | | S | | . | ilike | ILike | S | | . | in [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#predicate-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#predicate-functions"
- },"999": {
+ },"1001": {
"doc": "velox backend scalar function support",
"title": "String Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | ascii | Ascii | S | | . | base64 | Base64 | PS | base64 with chunkBase64String disabled is not supported | . | bit_length | BitLength | S | | . | btrim | StringTrimBoth | S | | . | char | Chr | S | | . | char_length | Length | S | | . | character_length | Length | S | | . | chr | Chr | S | | . | concat_ws | ConcatWs | S | | . | contains | ContainsExpressionBuilder | PS | BinaryType unsuppo [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#string-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#string-functions"
- },"1000": {
+ },"1002": {
"doc": "velox backend scalar function support",
"title": "Struct Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | named_struct | CreateNamedStruct | S | | . | struct | | S | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#struct-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#struct-functions"
- },"1001": {
+ },"1003": {
"doc": "velox backend scalar function support",
"title": "URL Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | parse_url | ParseUrl | | | . | url_decode | UrlDecode | S | | . | url_encode | UrlEncode | S | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#url-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#url-functions"
- },"1002": {
+ },"1004": {
"doc": "velox backend scalar function support",
"title": "XML Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | xpath | XPathList | | | . | xpath_boolean | XPathBoolean | | | . | xpath_double | XPathDouble | | | . | xpath_float | XPathFloat | | | . | xpath_int | XPathInt | | | . | xpath_long | XPathLong | | | . | xpath_number | XPathDouble | | | . | xpath_short | XPathShort | | | . | xpath_string | XPathString | | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#xml-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html#xml-functions"
- },"1003": {
+ },"1005": {
"doc": "velox backend scalar function support",
"title": "velox backend scalar function support",
"content": " ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html"
- },"1004": {
+ },"1006": {
"doc": "velox backend support progress",
"title": "The Operators and Functions Support Progress",
"content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. We use some notations t [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-support-progress.html#the-operators-and-functions-support-progress",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-support-progress.html#the-operators-and-functions-support-progress"
- },"1005": {
+ },"1007": {
"doc": "velox backend support progress",
"title": "Operator Map",
"content": "Gluten supports 30+ operators (Drag to right to see all data types) . | Executor | Description | Gluten Name | Velox Name | BOOLEAN | BYTE | SHORT | INT | LONG | FLOAT | DOUBLE | STRING | NULL | BINARY | ARRAY | MAP | STRUCT(ROW) | DATE | TIMESTAMP | DECIMAL | CALENDAR | UDT | . | FileSourceScanExec | Reading data from files, often from Hive tables | FileSourceScanExecTransformer | TableScanNode | S | S | S | S | S | S | S | S | S | S | NS | NS | NS | S | NS | NS | NS | N [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-support-progress.html#operator-map",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-support-progress.html#operator-map"
- },"1006": {
+ },"1008": {
"doc": "velox backend support progress",
"title": "Function Support Status",
"content": "Spark categorizes built-in functions into four types: Scalar Functions, Aggregate Functions, Window Functions, and Generator Functions. In Gluten, function support is automatically generated by a script and maintained in separate files. When running the script, the --spark_home arg should be set to either: . | The directory containing the Spark source code for the latest supported Spark version in Gluten, and the Spark project must be built from source. | Or use the insta [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-support-progress.html#function-support-status",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-support-progress.html#function-support-status"
- },"1007": {
+ },"1009": {
"doc": "velox backend support progress",
"title": "velox backend support progress",
"content": " ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-support-progress.html",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-support-progress.html"
- },"1008": {
+ },"1010": {
"doc": "velox backend troubleshooting",
"title": "Troubleshooting",
"content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
"url": "/docs/v1.5.0/velox-backend/velox-backend-troubleshooting.html#troubleshooting",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-troubleshooting.html#troubleshooting"
- },"1009": {
+ },"1011": {
"doc": "velox backend troubleshooting",
"title": "velox backend troubleshooting",
"content": " ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-troubleshooting.html",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-troubleshooting.html"
- },"1010": {
+ },"1012": {
"doc": "velox backend window function support",
"title": "Window Functions Support Status",
"content": "Out of 9 window functions in Spark 3.5, Gluten currently fully supports 9 functions. ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-window-function-support.html#window-functions-support-status",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-window-function-support.html#window-functions-support-status"
- },"1011": {
+ },"1013": {
"doc": "velox backend window function support",
"title": "Window Functions",
"content": "| Spark Functions | Spark Expressions | Status | Restrictions | . | cume_dist | CumeDist | S | | . | dense_rank | DenseRank | S | | . | lag | Lag | S | | . | lead | Lead | S | | . | nth_value | NthValue | S | | . | ntile | NTile | S | | . | percent_rank | PercentRank | S | | . | rank | Rank | S | | . | row_number | RowNumber | S | | . ",
"url": "/docs/v1.5.0/velox-backend/velox-backend-window-function-support.html#window-functions",
"relUrl": "/docs/v1.5.0/velox-backend/velox-backend-window-function-support.html#window-functions"
- },"1012": {
+ },"1014": {
"doc": "velox backend window function support",
"title": "velox backend window function support",
"content": " ",
diff --git a/output/blog/index.html b/output/blog/index.html
index d1706580..1de10593 100644
--- a/output/blog/index.html
+++ b/output/blog/index.html
@@ -89,6 +89,14 @@
<div class="page-content">
<ul class="blog-list">
+ <li>
+ <h2><a href="/2026/10/09/community-over-code-glasgow-2026.html">Community Over Code Glasgow 2026</a></h2>
+ <p class="post-meta">October 9, 2026 · Apache Gluten Community</p>
+ <h1 id="community-over-code-glasgow-2026">Community Over Code Glasgow 2026</h1>
+
+
+ </li>
+
<li>
<h2><a href="/2025/12/03/GlutenCon2025-poster.html">Glutencon2025 Poster</a></h2>
<p class="post-meta">December 3, 2025</p>
diff --git a/output/docs/developers/how-to-release/index.html b/output/docs/developers/how-to-release/index.html
index 64301888..de8b92b2 100644
--- a/output/docs/developers/how-to-release/index.html
+++ b/output/docs/developers/how-to-release/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } .site-nav > ul.nav-category-list > li > button svg { transform: rotate(-90deg); } .site-nav > ul.nav-category-list > li.nav-li [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } .site-nav > ul.nav-category-list > li > button svg { transform: rotate(-90deg); } .site-nav > ul.nav-category-list > li.nav-li [...]
<span class="nv">$ </span>gpg <span class="nt">--full-generate-key</span>
<span class="c"># list the GPG keys</span>
diff --git a/output/docs/v1.4.0/Configuration.html b/output/docs/v1.4.0/Configuration.html
index fb062226..53359ad8 100644
--- a/output/docs/v1.4.0/Configuration.html
+++ b/output/docs/v1.4.0/Configuration.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nv">sparkContext</span><span class="o">.</span><span class="py">setLocalProperty</span><span class="o">(</span><span class="n">key</span><span class="o">,</span> <span class="n">value</span><span class="o">)</span>
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"select * from demo_tables"</span><span class="o">).</span><span class="py">show</span><span class="o">()</span>
</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. All other marks mentioned may b [...]
diff --git a/output/docs/v1.4.0/developers/CppCodingStyle.html b/output/docs/v1.4.0/developers/CppCodingStyle.html
index f0e84ca6..69876309 100644
--- a/output/docs/v1.4.0/developers/CppCodingStyle.html
+++ b/output/docs/v1.4.0/developers/CppCodingStyle.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
pip3 install --user cmake-format
cmake-format --first-comment-is-literal True --in-place cpp/velox/CMakeLists.txt
</code></pre></div></div> <h2 id="naming-conventions"> <a href="#naming-conventions" class="anchor-heading" aria-labelledby="naming-conventions"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Naming Conventions </h2> <ul> <li>Use <strong>PascalCase</strong> for types (class, struct, enum, type alias, type template parameter) and file name.</li> <li>Use <strong>camelCase</strong> for function, member and local variable, and non-type template param [...]
diff --git a/output/docs/v1.4.0/developers/HowTo.html b/output/docs/v1.4.0/developers/HowTo.html
index 0bcbd0c5..a4473ba1 100644
--- a/output/docs/v1.4.0/developers/HowTo.html
+++ b/output/docs/v1.4.0/developers/HowTo.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> <ol> <li>Build Gluten and generate the example files:</li> </ol> <div class="language- [...]
mvn test -Pspark-3.2 -Pbackends-velox -pl backends-velox \
-am -DtagsToInclude="org.apache.gluten.tags.GenerateExample" \
diff --git a/output/docs/v1.4.0/developers/MicroBenchmarks.html b/output/docs/v1.4.0/developers/MicroBenchmarks.html
index 71a93342..07e87076 100644
--- a/output/docs/v1.4.0/developers/MicroBenchmarks.html
+++ b/output/docs/v1.4.0/developers/MicroBenchmarks.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3.</span>
diff --git a/output/docs/v1.4.0/developers/NewToGluten.html b/output/docs/v1.4.0/developers/NewToGluten.html
index f354da5f..d6a9ef78 100644
--- a/output/docs/v1.4.0/developers/NewToGluten.html
+++ b/output/docs/v1.4.0/developers/NewToGluten.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html b/output/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html
index bdd743f1..142f2921 100644
--- a/output/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html
+++ b/output/docs/v1.4.0/developers/ProfileMemoryOfGlutenWithVelox.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">tar </span>xzvf gperftools-<version>.tar.gz
<span class="nb">cd </span>gperftools-<version>
./configure
diff --git a/output/docs/v1.4.0/developers/QueryTrace.html b/output/docs/v1.4.0/developers/QueryTrace.html
index 5a173b24..579fea2e 100644
--- a/output/docs/v1.4.0/developers/QueryTrace.html
+++ b/output/docs/v1.4.0/developers/QueryTrace.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
├── conf_35_0.ini
├── data_35_0_0.parquet
├── plan_35_0.json
diff --git a/output/docs/v1.4.0/developers/SubstraitModifications.html b/output/docs/v1.4.0/developers/SubstraitModifications.html
index ddcfd54b..7e9977bc 100644
--- a/output/docs/v1.4.0/developers/SubstraitModifications.html
+++ b/output/docs/v1.4.0/developers/SubstraitModifications.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.4.0/developers/UsingGperftoolsInCH.html b/output/docs/v1.4.0/developers/UsingGperftoolsInCH.html
index 74f44a7f..d6ddd615 100644
--- a/output/docs/v1.4.0/developers/UsingGperftoolsInCH.html
+++ b/output/docs/v1.4.0/developers/UsingGperftoolsInCH.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
export LD_PRELOAD=$tcmalloc_path,libch.so # load the library in the driver
--files $tcmalloc_path # upload the library to the cluster
--conf spark.executorEnv.LD_PRELOAD=./libtcmalloc_and_profiler.so,libch.so # load the library in the executor
diff --git a/output/docs/v1.4.0/developers/UsingJemallocWithCH.html b/output/docs/v1.4.0/developers/UsingJemallocWithCH.html
index b050cffd..caee68b3 100644
--- a/output/docs/v1.4.0/developers/UsingJemallocWithCH.html
+++ b/output/docs/v1.4.0/developers/UsingJemallocWithCH.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
# At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracking.
#
# ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1).
diff --git a/output/docs/v1.4.0/developers/VeloxUDF.html b/output/docs/v1.4.0/developers/VeloxUDF.html
index ee57c3b7..899e9674 100644
--- a/output/docs/v1.4.0/developers/VeloxUDF.html
+++ b/output/docs/v1.4.0/developers/VeloxUDF.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
#include <velox/udf/Udf.h>
namespace {
diff --git a/output/docs/v1.4.0/developers/clickhouse-backend-debug.html b/output/docs/v1.4.0/developers/clickhouse-backend-debug.html
index e8d0996a..aae7f321 100644
--- a/output/docs/v1.4.0/developers/clickhouse-backend-debug.html
+++ b/output/docs/v1.4.0/developers/clickhouse-backend-debug.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <p>libch.so will be generated in <code class="language-plaintext highlighter-rouge">cpp-ch/build/utils/extern-local-engine/libch.so</code>.</p> </li> <li>Maven Build Gluten ClickHouse with Profile <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>mvn clean install -DskipTests -P delta -Pbackends-clickhouse -Pspark-3.3 -Pspark-ut
</code></pre></div> </div> </li> <li> <p>Set Maven Profiles in IntelliJ IDEA</p> <p><img src="../image/ClickHouse/gluten-debug-profile-settings.png" alt="gluten-debug-profile-settings.png" /></p> </li> <li> <p>Set Debug Configuration in IntelliJ IDEA</p> <p>For example, debug GlutenMathExpressionsSuite. <img src="../image/ClickHouse/gluten-debug-idea-config.png" alt="gluten-debug-idea-config.png" /></p> <p>VM Options: <code class="language-plaintext highlighter-rouge">-Dgluten.test.data. [...]
diff --git a/output/docs/v1.4.0/developers/improvement-proposals.html b/output/docs/v1.4.0/developers/improvement-proposals.html
index 21761f21..d4369d22 100644
--- a/output/docs/v1.4.0/developers/improvement-proposals.html
+++ b/output/docs/v1.4.0/developers/improvement-proposals.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.4.0/developers/velox-backend-CI.html b/output/docs/v1.4.0/developers/velox-backend-CI.html
index fb95a1fa..43d5886d 100644
--- a/output/docs/v1.4.0/developers/velox-backend-CI.html
+++ b/output/docs/v1.4.0/developers/velox-backend-CI.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.4.0/developers/velox-backend-build-in-docker.html b/output/docs/v1.4.0/developers/velox-backend-build-in-docker.html
index f76421e1..08ea0759 100644
--- a/output/docs/v1.4.0/developers/velox-backend-build-in-docker.html
+++ b/output/docs/v1.4.0/developers/velox-backend-build-in-docker.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
librt.so.1
libpthread.so.0
libdl.so.2
diff --git a/output/docs/v1.4.0/developers/velox-function-development-guide.html b/output/docs/v1.4.0/developers/velox-function-development-guide.html
index 696be5eb..13ae8105 100644
--- a/output/docs/v1.4.0/developers/velox-function-development-guide.html
+++ b/output/docs/v1.4.0/developers/velox-function-development-guide.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/docs/v1.4.0/get-started/ClickHouse.html b/output/docs/v1.4.0/get-started/ClickHouse.html
index 36116280..d87397d7 100644
--- a/output/docs/v1.4.0/get-started/ClickHouse.html
+++ b/output/docs/v1.4.0/get-started/ClickHouse.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker build <span class="nb">.</span> <span class="nt">-t</span> libch_builder
</code></pre></div></div> <p>You can also refer to <a href="https://clickhouse.com/docs/en/development/build/">How-to-Build-ClickHouse-on-Linux</a>.</p> <h5 id="for-compiling-gluten"> <a href="#for-compiling-gluten" class="anchor-heading" aria-labelledby="for-compiling-gluten"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> For compiling gluten </h5> <p>You need to install the following software manually:</p> <ul> <li>Java 8</li> <li>Maven 3.6.3 o [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <h5 id="compile-clickhouse-backend"> <a href="#compile-clickhouse-backend" class="anchor-heading" aria-labelledby [...]
diff --git a/output/docs/v1.4.0/get-started/GlutenUI.html b/output/docs/v1.4.0/get-started/GlutenUI.html
index e2c5c87c..dc221171 100644
--- a/output/docs/v1.4.0/get-started/GlutenUI.html
+++ b/output/docs/v1.4.0/get-started/GlutenUI.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.4.0/get-started/Velox.html b/output/docs/v1.4.0/get-started/Velox.html
index 6bf2241e..e883e3e3 100644
--- a/output/docs/v1.4.0/get-started/Velox.html
+++ b/output/docs/v1.4.0/get-started/Velox.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
diff --git a/output/docs/v1.4.0/get-started/VeloxABFS.html b/output/docs/v1.4.0/get-started/VeloxABFS.html
index fbebd572..4739866c 100644
--- a/output/docs/v1.4.0/get-started/VeloxABFS.html
+++ b/output/docs/v1.4.0/get-started/VeloxABFS.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
diff --git a/output/docs/v1.4.0/get-started/VeloxGCS.html b/output/docs/v1.4.0/get-started/VeloxGCS.html
index 498e2441..c94dc247 100644
--- a/output/docs/v1.4.0/get-started/VeloxGCS.html
+++ b/output/docs/v1.4.0/get-started/VeloxGCS.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-sh highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage.roo [...]
</code></pre></div></div> <h2 id="configuring-gcs-max-retry-count"> <a href="#configuring-gcs-max-retry-count" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-count"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry count </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected.</p> <div class="language-sh highlighter [...]
diff --git a/output/docs/v1.4.0/get-started/VeloxHBM.html b/output/docs/v1.4.0/get-started/VeloxHBM.html
index 385eded5..e71dbb42 100644
--- a/output/docs/v1.4.0/get-started/VeloxHBM.html
+++ b/output/docs/v1.4.0/get-started/VeloxHBM.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <p>After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">cd</span> /path/to/gluten
<span class="c">## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.3 and 3.5.1.</span>
diff --git a/output/docs/v1.4.0/get-started/VeloxIAA.html b/output/docs/v1.4.0/get-started/VeloxIAA.html
index 64f2db88..4c782a24 100644
--- a/output/docs/v1.4.0/get-started/VeloxIAA.html
+++ b/output/docs/v1.4.0/get-started/VeloxIAA.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">sudo </span>usermod <span class="nt">-aG</span> iaa username <span class="c"># need to relogin</span>
<span class="nb">sudo chgrp</span> <span class="nt">-R</span> iaa /dev/iax
<span class="nb">sudo chmod</span> <span class="nt">-R</span> g+rw /dev/iax
diff --git a/output/docs/v1.4.0/get-started/VeloxIceberg.html b/output/docs/v1.4.0/get-started/VeloxIceberg.html
index 28da0e03..f8d1e336 100644
--- a/output/docs/v1.4.0/get-started/VeloxIceberg.html
+++ b/output/docs/v1.4.0/get-started/VeloxIceberg.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <p>PartialOffload</p> <p>The write is fallback while read is offload.</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>INSERT INTO local.db.table SELECT id, data FROM source WHERE length(data) = 1;
</code></pre></div></div> <h2 id="reading"> <a href="#reading" class="anchor-heading" aria-labelledby="reading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Reading </h2> <h3 id="read-data"> <a href="#read-data" class="anchor-heading" aria-labelledby="read-data"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Read data </h3> <p>Offload/Fallback</p> <div class="table-wrapper"><table> <thead> <tr> <th>Tabl [...]
FROM local.db.table
diff --git a/output/docs/v1.4.0/get-started/VeloxLocalCache.html b/output/docs/v1.4.0/get-started/VeloxLocalCache.html
index 3e98892d..1cbaff96 100644
--- a/output/docs/v1.4.0/get-started/VeloxLocalCache.html
+++ b/output/docs/v1.4.0/get-started/VeloxLocalCache.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.gluten.sql.columnar.backend.velox.memCacheSize // the total size of in-mem cache, default is 128MB.
spark.gluten.sql.columnar.backend.velox.ssdCachePath // the folder to store the cache files, default is "/tmp".
spark.gluten.sql.columnar.backend.velox.ssdCacheSize // the total size of the SSD cache, default is 128MB. Velox will do in-mem cache only if this value is 0.
diff --git a/output/docs/v1.4.0/get-started/VeloxQAT.html b/output/docs/v1.4.0/get-started/VeloxQAT.html
index b47e5135..5b5640ed 100644
--- a/output/docs/v1.4.0/get-started/VeloxQAT.html
+++ b/output/docs/v1.4.0/get-started/VeloxQAT.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">source</span> ~/.bashrc
<span class="c"># Also set for root if running as non-root user</span>
diff --git a/output/docs/v1.4.0/get-started/VeloxS3.html b/output/docs/v1.4.0/get-started/VeloxS3.html
index f7851fec..5d0da22f 100644
--- a/output/docs/v1.4.0/get-started/VeloxS3.html
+++ b/output/docs/v1.4.0/get-started/VeloxS3.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
diff --git a/output/docs/v1.4.0/get-started/VeloxStageResourceAdj.html b/output/docs/v1.4.0/get-started/VeloxStageResourceAdj.html
index b5fbf263..bcb56265 100644
--- a/output/docs/v1.4.0/get-started/VeloxStageResourceAdj.html
+++ b/output/docs/v1.4.0/get-started/VeloxStageResourceAdj.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> </li> <li><strong>Enable Executor Dynamic Allocation</strong>: <div class="language-properties highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="py">spark.dynamicAllocation.enabled</span><span class="p">=</span><span class="s">true </span>
</code></pre></div> </div> </li> <li><strong>Resource Scheduler Compatibility</strong>:<br /> Ensure the underlying cluster resource manager (e.g., YARN, Kubernetes) supports dynamic resource allocation.</li> </ol> <h3 id="key-configurations"> <a href="#key-configurations" class="anchor-heading" aria-labelledby="key-configurations"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> <strong>Key Configurations</strong> </h3> <p>Add the following config [...]
</code></pre></div></div> <h3 id="how-it-works"> <a href="#how-it-works" class="anchor-heading" aria-labelledby="how-it-works"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> <strong>How It Works</strong> </h3> <p>The framework analyzes each stage during query planning and adjusts resource profiles in following scenarios:</p> <h4 id="scenario-1-fallback-operators-exist"> <a href="#scenario-1-fallback-operators-exist" class="anchor-heading" aria-la [...]
diff --git a/output/docs/v1.4.0/get-started/build-guide.html b/output/docs/v1.4.0/get-started/build-guide.html
index ed584ef1..6415165d 100644
--- a/output/docs/v1.4.0/get-started/build-guide.html
+++ b/output/docs/v1.4.0/get-started/build-guide.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.4.0/velox-backend/velox-backend-limitations.html b/output/docs/v1.4.0/velox-backend/velox-backend-limitations.html
index 17cbc651..79782cc0 100644
--- a/output/docs/v1.4.0/velox-backend/velox-backend-limitations.html
+++ b/output/docs/v1.4.0/velox-backend/velox-backend-limitations.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="filesource-format"> <a href="#filesource-format" class="anchor-heading" aria-labelledby="filesource-format"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> FileSource format </h4> <p>Currently, Gluten only fully supports parquet file format and partially support ORC. If other format is used, scan operator falls back to vanilla spark.</p> <h4 id="partitioned-table-scan"> <a href="#partitioned-table-scan" class="anc [...]
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e"</span><span class="o">)</span>
</code></pre></div></div> <p>Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><spa [...]
diff --git a/output/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html b/output/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html
index 72a7efe3..2bf7dd58 100644
--- a/output/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html
+++ b/output/docs/v1.4.0/velox-backend/velox-backend-scalar-function-support.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.4.0/velox-backend/velox-backend-support-progress.html b/output/docs/v1.4.0/velox-backend/velox-backend-support-progress.html
index 97fbcc2a..18b28822 100644
--- a/output/docs/v1.4.0/velox-backend/velox-backend-support-progress.html
+++ b/output/docs/v1.4.0/velox-backend/velox-backend-support-progress.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <p>Please check the links below for the detailed support status of each category:</p> <p><a href="./velox-backend-scalar-function-support.md">Scalar Functions Support Status</a></p> <h3 id="other-functions-support-status-to-be-updated"> <a href="#other-functions-support-status-to-be-updated" class="anchor-heading" aria-labelledby="other-functions-support-status-to-be-updated"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a [...]
diff --git a/output/docs/v1.4.0/velox-backend/velox-backend-troubleshooting.html b/output/docs/v1.4.0/velox-backend/velox-backend-troubleshooting.html
index 6e3ed049..2f0b2fee 100644
--- a/output/docs/v1.4.0/velox-backend/velox-backend-troubleshooting.html
+++ b/output/docs/v1.4.0/velox-backend/velox-backend-troubleshooting.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
# velox backend only
rm -rf $SPARK_HOME/jars/flatbuffers-*
rm -rf $SPARK_HOME/jars/arrow-*
diff --git a/output/docs/v1.5.0/Configuration.html b/output/docs/v1.5.0/Configuration.html
index 5de0c797..6c99232a 100644
--- a/output/docs/v1.5.0/Configuration.html
+++ b/output/docs/v1.5.0/Configuration.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/developers/CppCodingStyle.html b/output/docs/v1.5.0/developers/CppCodingStyle.html
index ec36bd72..aff6ba02 100644
--- a/output/docs/v1.5.0/developers/CppCodingStyle.html
+++ b/output/docs/v1.5.0/developers/CppCodingStyle.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
pip3 install --user cmake-format
cmake-format --first-comment-is-literal True --in-place cpp/velox/CMakeLists.txt
</code></pre></div></div> <h2 id="naming-conventions"> <a href="#naming-conventions" class="anchor-heading" aria-labelledby="naming-conventions"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Naming Conventions </h2> <ul> <li>Use <strong>PascalCase</strong> for types (class, struct, enum, type alias, type template parameter) and file name.</li> <li>Use <strong>camelCase</strong> for function, member and local variable, and non-type template param [...]
diff --git a/output/docs/v1.5.0/developers/HowTo.html b/output/docs/v1.5.0/developers/HowTo.html
index c78dad8d..c956af09 100644
--- a/output/docs/v1.5.0/developers/HowTo.html
+++ b/output/docs/v1.5.0/developers/HowTo.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> <ol> <li>Build Gluten and generate the example files:</li> </ol> <div class="language- [...]
mvn test -Pspark-3.2 -Pbackends-velox -pl backends-velox \
-am -DtagsToInclude="org.apache.gluten.tags.GenerateExample" \
diff --git a/output/docs/v1.5.0/developers/MicroBenchmarks.html b/output/docs/v1.5.0/developers/MicroBenchmarks.html
index 1b828eb1..06dd6ffb 100644
--- a/output/docs/v1.5.0/developers/MicroBenchmarks.html
+++ b/output/docs/v1.5.0/developers/MicroBenchmarks.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3.</span>
diff --git a/output/docs/v1.5.0/developers/NewToGluten.html b/output/docs/v1.5.0/developers/NewToGluten.html
index 92695121..56c2fcd2 100644
--- a/output/docs/v1.5.0/developers/NewToGluten.html
+++ b/output/docs/v1.5.0/developers/NewToGluten.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.executor.extraJavaOptions=-Dio.netty.tryReflectionSetAccessible=true
</code></pre></div></div> <h3 id="maven"> <a href="#maven" class="anchor-heading" aria-labelledby="maven"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Maven </h3> <p>Gluten requires Maven 3.6.3 or above.</p> <h3 id="gcc"> <a href="#gcc" class="anchor-heading" aria-labelledby="gcc"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> GCC </h3> <p>Gluten requires GCC 11 or above.</p> <h2 id="development"> <a hr [...]
gluten_home/dev/builddeps-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON <span class="nt">--build_type</span><span class="o">=</span>Debug
diff --git a/output/docs/v1.5.0/developers/PartialProject.html b/output/docs/v1.5.0/developers/PartialProject.html
index 9d61ec46..f947985d 100644
--- a/output/docs/v1.5.0/developers/PartialProject.html
+++ b/output/docs/v1.5.0/developers/PartialProject.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html b/output/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html
index 56b2861e..ab048af4 100644
--- a/output/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html
+++ b/output/docs/v1.5.0/developers/ProfileMemoryOfGlutenWithVelox.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">tar </span>xzvf gperftools-<version>.tar.gz
<span class="nb">cd </span>gperftools-<version>
./configure
diff --git a/output/docs/v1.5.0/developers/QueryTrace.html b/output/docs/v1.5.0/developers/QueryTrace.html
index 3b56e0a8..35d9b286 100644
--- a/output/docs/v1.5.0/developers/QueryTrace.html
+++ b/output/docs/v1.5.0/developers/QueryTrace.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
├── conf_35_0.ini
├── data_35_0_0.parquet
├── plan_35_0.json
diff --git a/output/docs/v1.5.0/developers/SubstraitModifications.html b/output/docs/v1.5.0/developers/SubstraitModifications.html
index a76ee3ab..b9daf84a 100644
--- a/output/docs/v1.5.0/developers/SubstraitModifications.html
+++ b/output/docs/v1.5.0/developers/SubstraitModifications.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/developers/UsingGperftoolsInCH.html b/output/docs/v1.5.0/developers/UsingGperftoolsInCH.html
index 1fc41100..88f71ef6 100644
--- a/output/docs/v1.5.0/developers/UsingGperftoolsInCH.html
+++ b/output/docs/v1.5.0/developers/UsingGperftoolsInCH.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
export LD_PRELOAD=$tcmalloc_path,libch.so # load the library in the driver
--files $tcmalloc_path # upload the library to the cluster
--conf spark.executorEnv.LD_PRELOAD=./libtcmalloc_and_profiler.so,libch.so # load the library in the executor
diff --git a/output/docs/v1.5.0/developers/UsingJemallocWithCH.html b/output/docs/v1.5.0/developers/UsingJemallocWithCH.html
index 2ca91bd9..6a0a4795 100644
--- a/output/docs/v1.5.0/developers/UsingJemallocWithCH.html
+++ b/output/docs/v1.5.0/developers/UsingJemallocWithCH.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
# At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracking.
#
# ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1).
diff --git a/output/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html b/output/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html
index e66c69bc..cf9e01e4 100644
--- a/output/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html
+++ b/output/docs/v1.5.0/developers/VeloxDynamicSizingOffheap.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/developers/VeloxUDF.html b/output/docs/v1.5.0/developers/VeloxUDF.html
index 6696266b..6e3da8c2 100644
--- a/output/docs/v1.5.0/developers/VeloxUDF.html
+++ b/output/docs/v1.5.0/developers/VeloxUDF.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
#include <velox/udf/Udf.h>
namespace {
diff --git a/output/docs/v1.5.0/developers/clickhouse-backend-debug.html b/output/docs/v1.5.0/developers/clickhouse-backend-debug.html
index 7cb2b373..e40869a4 100644
--- a/output/docs/v1.5.0/developers/clickhouse-backend-debug.html
+++ b/output/docs/v1.5.0/developers/clickhouse-backend-debug.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <p>libch.so will be generated in <code class="language-plaintext highlighter-rouge">cpp-ch/build/utils/extern-local-engine/libch.so</code>.</p> </li> <li>Maven Build Gluten ClickHouse with Profile <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>mvn clean install -DskipTests -P delta -Pbackends-clickhouse -Pspark-3.3 -Pspark-ut
</code></pre></div> </div> </li> <li> <p>Set Maven Profiles in IntelliJ IDEA</p> <p><img src="../image/ClickHouse/gluten-debug-profile-settings.png" alt="gluten-debug-profile-settings.png" /></p> </li> <li> <p>Set Debug Configuration in IntelliJ IDEA</p> <p>For example, debug GlutenMathExpressionsSuite. <img src="../image/ClickHouse/gluten-debug-idea-config.png" alt="gluten-debug-idea-config.png" /></p> <p>VM Options: <code class="language-plaintext highlighter-rouge">-Dgluten.test.data. [...]
diff --git a/output/docs/v1.5.0/developers/improvement-proposals.html b/output/docs/v1.5.0/developers/improvement-proposals.html
index 61eb5cff..63e9a262 100644
--- a/output/docs/v1.5.0/developers/improvement-proposals.html
+++ b/output/docs/v1.5.0/developers/improvement-proposals.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/developers/velox-backend-CI.html b/output/docs/v1.5.0/developers/velox-backend-CI.html
index c28044a3..b1efcd8e 100644
--- a/output/docs/v1.5.0/developers/velox-backend-CI.html
+++ b/output/docs/v1.5.0/developers/velox-backend-CI.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/developers/velox-backend-build-in-docker.html b/output/docs/v1.5.0/developers/velox-backend-build-in-docker.html
index 954d18a4..273e26d9 100644
--- a/output/docs/v1.5.0/developers/velox-backend-build-in-docker.html
+++ b/output/docs/v1.5.0/developers/velox-backend-build-in-docker.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
librt.so.1
libpthread.so.0
libdl.so.2
diff --git a/output/docs/v1.5.0/developers/velox-function-development-guide.html b/output/docs/v1.5.0/developers/velox-function-development-guide.html
index 27dbb0cd..fd85b874 100644
--- a/output/docs/v1.5.0/developers/velox-function-development-guide.html
+++ b/output/docs/v1.5.0/developers/velox-function-development-guide.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/docs/v1.5.0/get-started/ClickHouse.html b/output/docs/v1.5.0/get-started/ClickHouse.html
index 6a4705ab..e2fbb78b 100644
--- a/output/docs/v1.5.0/get-started/ClickHouse.html
+++ b/output/docs/v1.5.0/get-started/ClickHouse.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker build <span class="nb">.</span> <span class="nt">-t</span> libch_builder
</code></pre></div></div> <p>You can also refer to <a href="https://clickhouse.com/docs/en/development/build/">How-to-Build-ClickHouse-on-Linux</a>.</p> <h5 id="for-compiling-gluten"> <a href="#for-compiling-gluten" class="anchor-heading" aria-labelledby="for-compiling-gluten"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> For compiling gluten </h5> <p>You need to install the following software manually:</p> <ul> <li>Java 8</li> <li>Maven 3.6.3 o [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <h5 id="compile-clickhouse-backend"> <a href="#compile-clickhouse-backend" class="anchor-heading" aria-labelledby [...]
diff --git a/output/docs/v1.5.0/get-started/GlutenUI.html b/output/docs/v1.5.0/get-started/GlutenUI.html
index 173f2208..a5a3f30a 100644
--- a/output/docs/v1.5.0/get-started/GlutenUI.html
+++ b/output/docs/v1.5.0/get-started/GlutenUI.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/get-started/Velox.html b/output/docs/v1.5.0/get-started/Velox.html
index cbab2929..b8f69715 100644
--- a/output/docs/v1.5.0/get-started/Velox.html
+++ b/output/docs/v1.5.0/get-started/Velox.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
diff --git a/output/docs/v1.5.0/get-started/VeloxABFS.html b/output/docs/v1.5.0/get-started/VeloxABFS.html
index f5e21db4..ce086ab0 100644
--- a/output/docs/v1.5.0/get-started/VeloxABFS.html
+++ b/output/docs/v1.5.0/get-started/VeloxABFS.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
diff --git a/output/docs/v1.5.0/get-started/VeloxGCS.html b/output/docs/v1.5.0/get-started/VeloxGCS.html
index 3c9e4b56..f7dc0b44 100644
--- a/output/docs/v1.5.0/get-started/VeloxGCS.html
+++ b/output/docs/v1.5.0/get-started/VeloxGCS.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-sh highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage.roo [...]
</code></pre></div></div> <h2 id="configuring-gcs-max-retry-count"> <a href="#configuring-gcs-max-retry-count" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-count"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry count </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected.</p> <div class="language-sh highlighter [...]
diff --git a/output/docs/v1.5.0/get-started/VeloxGPU.html b/output/docs/v1.5.0/get-started/VeloxGPU.html
index 976dd007..033cb121 100644
--- a/output/docs/v1.5.0/get-started/VeloxGPU.html
+++ b/output/docs/v1.5.0/get-started/VeloxGPU.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">--name</span> gpu_gluten_container <span class="nt">--gpus</span> all <span class="nt">-it</span> apache/gluten:centos-9-jdk8-cudf
</code></pre></div></div> <ul> <li><strong>Image Includes</strong>: Native build cache, Gluten dependencies, Spark 3.4 environment.</li> </ul><hr /> <h2 id="5-build--deployment"> <a href="#5-build--deployment" class="anchor-heading" aria-labelledby="5-build--deployment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> <strong>5. Build & Deployment</strong> </h2> <h4 id="dependencies"> <a href="#dependencies" class="anchor-heading" aria-labelled [...]
</code></pre></div></div><hr /> <h2 id="6-gpu-operator-support-status"> <a href="#6-gpu-operator-support-status" class="anchor-heading" aria-labelledby="6-gpu-operator-support-status"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> <strong>6. GPU Operator Support Status</strong> </h2> <p>| <strong>Operator</strong> | <strong>Status</strong> | <strong>Notes</strong> |<br /> |—————–|—————–|————————–| | <strong>Scan</strong> | ❌ Not supported| In Dev [...]
diff --git a/output/docs/v1.5.0/get-started/VeloxIceberg.html b/output/docs/v1.5.0/get-started/VeloxIceberg.html
index 6661dde9..653c54fa 100644
--- a/output/docs/v1.5.0/get-started/VeloxIceberg.html
+++ b/output/docs/v1.5.0/get-started/VeloxIceberg.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <p>PartialOffload</p> <p>The write is fallback while read is offload.</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>INSERT INTO local.db.table SELECT id, data FROM source WHERE length(data) = 1;
</code></pre></div></div> <h2 id="reading"> <a href="#reading" class="anchor-heading" aria-labelledby="reading"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Reading </h2> <h3 id="read-data"> <a href="#read-data" class="anchor-heading" aria-labelledby="read-data"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Read data </h3> <p>Offload/Fallback</p> <div class="table-wrapper"><table> <thead> <tr> <th>Tabl [...]
FROM local.db.table
diff --git a/output/docs/v1.5.0/get-started/VeloxLocalCache.html b/output/docs/v1.5.0/get-started/VeloxLocalCache.html
index 034f7b87..83490678 100644
--- a/output/docs/v1.5.0/get-started/VeloxLocalCache.html
+++ b/output/docs/v1.5.0/get-started/VeloxLocalCache.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.gluten.sql.columnar.backend.velox.memCacheSize // the total size of in-mem cache, default is 128MB.
spark.gluten.sql.columnar.backend.velox.ssdCachePath // the folder to store the cache files, default is "/tmp".
spark.gluten.sql.columnar.backend.velox.ssdCacheSize // the total size of the SSD cache, default is 128MB. Velox will do in-mem cache only if this value is 0.
diff --git a/output/docs/v1.5.0/get-started/VeloxQAT.html b/output/docs/v1.5.0/get-started/VeloxQAT.html
index 3bda7259..888db396 100644
--- a/output/docs/v1.5.0/get-started/VeloxQAT.html
+++ b/output/docs/v1.5.0/get-started/VeloxQAT.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">source</span> ~/.bashrc
<span class="c"># Also set for root if running as non-root user</span>
diff --git a/output/docs/v1.5.0/get-started/VeloxS3.html b/output/docs/v1.5.0/get-started/VeloxS3.html
index 77fe2e64..bbbdf0f7 100644
--- a/output/docs/v1.5.0/get-started/VeloxS3.html
+++ b/output/docs/v1.5.0/get-started/VeloxS3.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
diff --git a/output/docs/v1.5.0/get-started/VeloxStageResourceAdj.html b/output/docs/v1.5.0/get-started/VeloxStageResourceAdj.html
index 0f657ac2..716e8f05 100644
--- a/output/docs/v1.5.0/get-started/VeloxStageResourceAdj.html
+++ b/output/docs/v1.5.0/get-started/VeloxStageResourceAdj.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> </li> <li><strong>Enable Executor Dynamic Allocation</strong>: <div class="language-properties highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="py">spark.dynamicAllocation.enabled</span><span class="p">=</span><span class="s">true </span>
</code></pre></div> </div> </li> <li><strong>Resource Scheduler Compatibility</strong>:<br /> Ensure the underlying cluster resource manager (e.g., YARN, Kubernetes) supports dynamic resource allocation.</li> </ol> <h3 id="key-configurations"> <a href="#key-configurations" class="anchor-heading" aria-labelledby="key-configurations"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> <strong>Key Configurations</strong> </h3> <p>Add the following config [...]
</code></pre></div></div> <h3 id="how-it-works"> <a href="#how-it-works" class="anchor-heading" aria-labelledby="how-it-works"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> <strong>How It Works</strong> </h3> <p>The framework analyzes each stage during query planning and adjusts resource profiles in following scenarios:</p> <h4 id="scenario-1-fallback-operators-exist"> <a href="#scenario-1-fallback-operators-exist" class="anchor-heading" aria-la [...]
diff --git a/output/docs/v1.5.0/get-started/build-guide.html b/output/docs/v1.5.0/get-started/build-guide.html
index 369cd049..5889afe8 100644
--- a/output/docs/v1.5.0/get-started/build-guide.html
+++ b/output/docs/v1.5.0/get-started/build-guide.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html b/output/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html
index 54ffde57..5b248b33 100644
--- a/output/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html
+++ b/output/docs/v1.5.0/velox-backend/velox-backend-aggregate-function-support.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html b/output/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html
index 6920a4a9..bb193518 100644
--- a/output/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html
+++ b/output/docs/v1.5.0/velox-backend/velox-backend-generator-function-support.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/velox-backend/velox-backend-limitations.html b/output/docs/v1.5.0/velox-backend/velox-backend-limitations.html
index 3778e1d1..f9f221af 100644
--- a/output/docs/v1.5.0/velox-backend/velox-backend-limitations.html
+++ b/output/docs/v1.5.0/velox-backend/velox-backend-limitations.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="filesource-format"> <a href="#filesource-format" class="anchor-heading" aria-labelledby="filesource-format"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> FileSource format </h4> <p>Currently, Gluten only fully supports parquet file format and partially support ORC. If other format is used, scan operator falls back to vanilla spark.</p> <h4 id="partitioned-table-scan"> <a href="#partitioned-table-scan" class="anc [...]
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e"</span><span class="o">)</span>
</code></pre></div></div> <p>Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><spa [...]
diff --git a/output/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html b/output/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html
index 1d6cce22..a4a3645d 100644
--- a/output/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html
+++ b/output/docs/v1.5.0/velox-backend/velox-backend-scalar-function-support.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/v1.5.0/velox-backend/velox-backend-support-progress.html b/output/docs/v1.5.0/velox-backend/velox-backend-support-progress.html
index 99000c84..45cd45b2 100644
--- a/output/docs/v1.5.0/velox-backend/velox-backend-support-progress.html
+++ b/output/docs/v1.5.0/velox-backend/velox-backend-support-progress.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
export spark_dir=/tmp/spark
export spark_version=3.5
diff --git a/output/docs/v1.5.0/velox-backend/velox-backend-troubleshooting.html b/output/docs/v1.5.0/velox-backend/velox-backend-troubleshooting.html
index 88513fa6..9557c223 100644
--- a/output/docs/v1.5.0/velox-backend/velox-backend-troubleshooting.html
+++ b/output/docs/v1.5.0/velox-backend/velox-backend-troubleshooting.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
# velox backend only
rm -rf $SPARK_HOME/jars/flatbuffers-*
rm -rf $SPARK_HOME/jars/arrow-*
diff --git a/output/docs/v1.5.0/velox-backend/velox-backend-window-function-support.html b/output/docs/v1.5.0/velox-backend/velox-backend-window-function-support.html
index f3ba5367..e034e74f 100644
--- a/output/docs/v1.5.0/velox-backend/velox-backend-window-function-support.html
+++ b/output/docs/v1.5.0/velox-backend/velox-backend-window-function-support.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/feed.xml b/output/feed.xml
index be8874e8..de64f799 100644
--- a/output/feed.xml
+++ b/output/feed.xml
@@ -1,4 +1,19 @@
-<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="3.9.3">Jekyll</generator><link href="https://gluten.apache.org/feed.xml" rel="self" type="application/atom+xml" /><link href="https://gluten.apache.org/" rel="alternate" type="text/html" /><updated>2026-09-24T11:27:07+00:00</updated><id>https://gluten.apache.org/feed.xml</id><title type="html">Apache Gluten</title><subtitle>Gluten is a middle layer responsible [...]
+<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="3.9.3">Jekyll</generator><link href="https://gluten.apache.org/feed.xml" rel="self" type="application/atom+xml" /><link href="https://gluten.apache.org/" rel="alternate" type="text/html" /><updated>2026-10-09T12:57:55+00:00</updated><id>https://gluten.apache.org/feed.xml</id><title type="html">Apache Gluten</title><subtitle>Gluten is a middle layer responsible [...]
+
+<p>Community over Code Glasgow is less than a month away! <strong>Oct 11–14</strong>, the Apache community gathers in Glasgow, Scotland.</p>
+
+<p>25+ projects are signed up for the hackathon, keynotes are lined up, and the schedule is packed across <strong>20 tracks</strong>. Registration is still open but time is running out — book your travel now. This is where the people behind the projects you depend on come together.</p>
+
+<p>Apache Gluten will be represented with <strong>2 sessions</strong> at the conference — come find us there!</p>
+
+<ul>
+ <li><a href="https://web.cvent.com/event/ac71ce47-2b5f-424c-abfe-5b48255315fb/summary?RefId=XSummary&amp;session=31f3bd3d-f0e0-4098-be49-022515872f63">Latest Apache Gluten Optimizations: From Planning to Execution</a></li>
+ <li><a href="https://web.cvent.com/event/ac71ce47-2b5f-424c-abfe-5b48255315fb/summary?RefId=XSummary&amp;session=97bc8b56-542c-4ddb-ae5d-11003b3a4717">Apache Gluten: From Incubating to Top-Level Project — Architecture, Ecosystem, and Latest Advances</a></li>
+</ul>
+
+<p>🔗 <a href="https://communityovercode.apache.org/events/glasgow-2026/">https://communityovercode.apache.org/events/glasgow-2026/</a></p>
+
+<p>#CommunityOverCode #ASF2026Glasgow #OpenSource</p></content><author><name>Apache Gluten Community</name></author><summary type="html">Community Over Code Glasgow 2026</summary></entry><entry><title type="html">Glutencon2025 Poster</title><link href="https://gluten.apache.org/2025/12/03/GlutenCon2025-poster.html" rel="alternate" type="text/html" title="Glutencon2025 Poster" /><published>2025-12-03T00:00:00+00:00</published><updated>2025-12-03T00:00:00+00:00</updated><id [...]
<img width="800" alt="image" src="/assets/images/GlutenCon2025/poster.png" />
</div></content><author><name></name></author><summary type="html"></summary></entry><entry><title type="html">Glutencon2025 Chinese</title><link href="https://gluten.apache.org/2025/11/06/GlutenCon2025-Chinese.html" rel="alternate" type="text/html" title="Glutencon2025 Chinese" /><published>2025-11-06T00:00:00+00:00</published><updated>2025-11-06T00:00:00+00:00</updated><id>https://gluten.apache.org/2025/11/06/GlutenCon2025-Chinese</id><content type="html" xml:base="https://gluten [...]
diff --git a/output/index.html b/output/index.html
index 293b6927..95239351 100644
--- a/output/index.html
+++ b/output/index.html
@@ -247,6 +247,11 @@
<h2 class="section-title">Latest News</h2>
<div class="news-grid">
+ <a href="/2026/10/09/community-over-code-glasgow-2026.html" class="news-card">
+ <div class="news-date">October 9, 2026</div>
+ <h3>Community Over Code Glasgow 2026</h3>
+ </a>
+
<a href="/2025/12/03/GlutenCon2025-poster.html" class="news-card">
<div class="news-date">December 3, 2025</div>
<h3>Glutencon2025 Poster</h3>
@@ -257,11 +262,6 @@
<h3>Glutencon2025 English</h3>
</a>
- <a href="/2025/11/06/GlutenCon2025-Chinese.html" class="news-card">
- <div class="news-date">November 6, 2025</div>
- <h3>Glutencon2025 Chinese</h3>
- </a>
-
</div>
<p style="margin-top:32px;"><a href="/blog/" class="btn btn-secondary" style="display:inline-flex;">All posts →</a></p>
</div>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org