You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by gi...@apache.org on 2025/06/16 10:25:21 UTC
(incubator-gluten-site) branch asf-site updated: Automatic Site Publish by Buildbot
This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/incubator-gluten-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 22b88d7 Automatic Site Publish by Buildbot
22b88d7 is described below
commit 22b88d790e817909d6de9cb1fe085e01ccced095
Author: buildbot <us...@infra.apache.org>
AuthorDate: Mon Jun 16 10:25:18 2025 +0000
Automatic Site Publish by Buildbot
---
output/2024/09/03/Gluten-1.2.0-release.html | 2 +-
output/404.html | 2 +-
output/about/index.html | 2 +-
output/archives/index.html | 2 +-
.../v1.1.1/developers/UsingGperftoolsInCH.html | 6 -
.../v1.1.1/developers/UsingJemallocWithCH.html | 12 -
.../v1.1.1/developers/cpp_code_style/index.html | 2 +-
.../v1.1.1/developers/docker_centos7/index.html | 2 +-
.../archives/v1.1.1/developers/docker_centos8.html | 2 +-
.../v1.1.1/developers/docker_ubuntu/index.html | 2 +-
output/archives/v1.1.1/developers/howto/index.html | 4 +-
.../v1.1.1/developers/howtorelease/index.html | 2 +-
output/archives/v1.1.1/developers/index.html | 2 +-
.../v1.1.1/developers/microbenchmark/index.html | 2 +-
output/archives/v1.1.1/developers/newto/index.html | 2 +-
.../v1.1.1/developers/substrait/index.html | 2 +-
.../velox_function_development/index.html | 2 +-
.../docs/clickhouse/getting-started/index.html | 2 +-
output/archives/v1.1.1/docs/clickhouse/index.html | 2 +-
output/archives/v1.1.1/docs/index.html | 2 +-
.../v1.1.1/docs/velox/build_parameters/index.html | 2 +-
output/archives/v1.1.1/docs/velox/gcs/index.html | 2 +-
.../v1.1.1/docs/velox/getting-started/index.html | 4 +-
output/archives/v1.1.1/docs/velox/index.html | 2 +-
output/archives/v1.1.1/docs/velox/s3/index.html | 2 +-
output/archives/v1.1.1/index.html | 2 +-
.../archives/v1.2.1/docs/configuration/index.html | 2 +-
.../docs/developers/UsingGperftoolsInCH.html | 6 -
.../docs/developers/UsingJemallocWithCH.html | 12 -
.../docs/developers/cpp-code-style/index.html | 2 +-
.../docs/developers/docker-centos7/index.html | 2 +-
.../docs/developers/docker-centos8/index.html | 2 +-
.../docs/developers/docker-ubuntu/index.html | 2 +-
.../docs/developers/how-to-release/index.html | 2 +-
.../developers/how-to-security-scan/index.html | 2 +-
.../v1.2.1/docs/developers/how-to-use/index.html | 2 +-
output/archives/v1.2.1/docs/developers/index.html | 2 +-
.../docs/developers/microbenchmarks/index.html | 2 +-
.../v1.2.1/docs/developers/new-to/index.html | 2 +-
.../v1.2.1/docs/developers/substrait/index.html | 2 +-
.../docs/developers/velox-function-dev/index.html | 2 +-
.../v1.2.1/docs/getting-started/abfs/index.html | 4 +-
.../docs/getting-started/build-guide/index.html | 2 +-
.../getting-started/clickhouse-backend/index.html | 2 +-
.../v1.2.1/docs/getting-started/gcs/index.html | 2 +-
.../v1.2.1/docs/getting-started/index.html | 2 +-
.../docs/getting-started/localcache/index.html | 2 +-
.../v1.2.1/docs/getting-started/s3/index.html | 4 +-
.../docs/getting-started/velox-backend/index.html | 10 +-
output/archives/v1.2.1/docs/index.html | 2 +-
.../archives/v1.2.1/docs/velox-backend/index.html | 2 +-
.../docs/velox-backend/limitations/index.html | 2 +-
.../v1.2.1/docs/velox-backend/support/index.html | 2 +-
.../docs/velox-backend/troubleshooting/index.html | 2 +-
.../v1.2.1/docs/velox-backend/udf/index.html | 2 +-
output/archives/v1.2.1/index.html | 2 +-
output/archives/v1.3.0/configuration.html | 4 +
output/archives/v1.3.0/developers.html | 1 +
.../v1.3.0/developers/cpp-code-style/index.html | 1 +
.../v1.3.0/developers/docker-centos7/index.html | 47 +
.../v1.3.0/developers/docker-centos8/index.html | 35 +
.../v1.3.0/developers/docker-ubuntu/index.html | 44 +
.../developers/how-to-security-scan/index.html | 2 +
.../v1.3.0/developers/how-to-use/index.html | 22 +
.../v1.3.0}/developers/microbenchmarks/index.html | 2 +-
.../v1.3.0}/developers/new-to/index.html | 2 +-
.../v1.3.0/developers/substrait/index.html | 1 +
.../developers/velox-function-dev/index.html | 11 +
output/archives/v1.3.0/getting-started.html | 1 +
.../v1.3.0/getting-started/abfs/index.html | 2 +
.../v1.3.0/getting-started/build-guide/index.html | 1 +
.../getting-started/clickhouse-backend/index.html | 2 +-
.../archives/v1.3.0/getting-started/gcs/index.html | 6 +
.../v1.3.0/getting-started/localcache/index.html | 8 +
.../archives/v1.3.0/getting-started/s3/index.html | 11 +
.../getting-started/velox-backend/index.html | 10 +-
output/archives/v1.3.0/index.html | 1 +
output/archives/v1.3.0/velox-backend.html | 1 +
.../v1.3.0/velox-backend/limitations/index.html | 24 +
.../v1.3.0/velox-backend/support/index.html | 1 +
.../velox-backend/troubleshooting/index.html | 46 +
.../archives/v1.3.0/velox-backend/udf/index.html | 87 +
output/assets/js/search-data.json | 4695 ++++++++++----------
output/blog/index.html | 2 +-
output/contact/index.html | 2 +-
output/contributing/index.html | 2 +-
output/docs/configuration/index.html | 4 -
output/docs/developers/UsingGperftoolsInCH.html | 6 -
output/docs/developers/UsingJemallocWithCH.html | 12 -
output/docs/developers/cpp-code-style/index.html | 1 -
output/docs/developers/docker-centos7/index.html | 47 -
output/docs/developers/docker-centos8/index.html | 35 -
output/docs/developers/docker-ubuntu/index.html | 44 -
output/docs/developers/how-to-release/index.html | 2 +-
.../developers/how-to-security-scan/index.html | 2 -
output/docs/developers/how-to-use/index.html | 22 -
output/docs/developers/index.html | 1 -
output/docs/developers/substrait/index.html | 1 -
.../docs/developers/velox-function-dev/index.html | 11 -
output/docs/getting-started/abfs/index.html | 2 -
output/docs/getting-started/build-guide/index.html | 1 -
output/docs/getting-started/gcs/index.html | 6 -
output/docs/getting-started/index.html | 1 -
output/docs/getting-started/localcache/index.html | 8 -
output/docs/getting-started/s3/index.html | 11 -
output/docs/index.html | 1 -
output/docs/latest.html | 6 +
.../v1.1.1/developers/UsingGperftoolsInCH.md | 0
.../v1.1.1/developers/UsingJemallocWithCH.md | 0
.../v1.2.1/docs/developers/UsingGperftoolsInCH.md | 0
.../v1.2.1/docs/developers/UsingJemallocWithCH.md | 0
.../docs/get-started/Work-with-pyspark.ipynb | 0
.../{ => v1.3.0}/developers/UsingGperftoolsInCH.md | 0
.../{ => v1.3.0}/developers/UsingJemallocWithCH.md | 0
.../get-started/Work-with-pyspark.ipynb | 0
output/docs/velox-backend/index.html | 1 -
output/docs/velox-backend/limitations/index.html | 24 -
output/docs/velox-backend/support/index.html | 1 -
.../docs/velox-backend/troubleshooting/index.html | 46 -
output/docs/velox-backend/udf/index.html | 87 -
output/downloads/index.html | 2 +-
output/feed.xml | 2 +-
output/index.html | 2 +-
.../update/2024/02/08/welcome-to-jekyll.html | 2 +-
output/poweredby/index.html | 2 +-
output/references/index.html | 2 +-
126 files changed, 2792 insertions(+), 2833 deletions(-)
diff --git a/output/2024/09/03/Gluten-1.2.0-release.html b/output/2024/09/03/Gluten-1.2.0-release.html
index 45fba37..d9e0357 100644
--- a/output/2024/09/03/Gluten-1.2.0-release.html
+++ b/output/2024/09/03/Gluten-1.2.0-release.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } </style> <script src="/assets/js/vendor/lunr.min.js"></script> <script src="/assets/js/just-the-docs.js"></script> <meta name= [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } .site-nav > ul.nav-category-list > li > button svg { transform: rotate(-90deg); } .site-nav > ul.nav-category-list > li.nav-li [...]
diff --git a/output/404.html b/output/404.html
index d1d7e91..56653be 100644
--- a/output/404.html
+++ b/output/404.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } </style> <script src="/assets/js/vendor/lunr.min.js"></script> <script src="/assets/js/just-the-docs.js"></script> <meta name= [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } .site-nav > ul.nav-category-list > li > button svg { transform: rotate(-90deg); } .site-nav > ul.nav-category-list > li.nav-li [...]
diff --git a/output/about/index.html b/output/about/index.html
index e1040e6..fdbec64 100644
--- a/output/about/index.html
+++ b/output/about/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(10)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:n [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(8)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
diff --git a/output/archives/index.html b/output/archives/index.html
index bf33ad5..43edaa8 100644
--- a/output/archives/index.html
+++ b/output/archives/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(3)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li a { background-image: none; } .site-nav > ul:not [...]
diff --git a/output/archives/v1.1.1/developers/UsingGperftoolsInCH.html b/output/archives/v1.1.1/developers/UsingGperftoolsInCH.html
deleted file mode 100644
index 7015d53..0000000
--- a/output/archives/v1.1.1/developers/UsingGperftoolsInCH.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } </style> <script src="/assets/js/vendor/lunr.min.js"></script> <script src="/assets/js/just-the-docs.js"></script> <meta name= [...]
-export LD_PRELOAD=$tcmalloc_path,libch.so # load the library in the driver
---files $tcmalloc_path # upload the library to the cluster
---conf spark.executorEnv.LD_PRELOAD=./libtcmalloc_and_profiler.so,libch.so # load the library in the executor
---conf spark.executorEnv.HEAPPROFILE=/tmp/gluten_heap_perf # set the heap profile path, you can change to CPUPROFILE for CPU profiling
-</code></pre></div></div> <p>For thrift server on local machine, note using <code class="language-plaintext highlighter-rouge">export LD_PRELOAD="$tcmalloc_path libch.so" # load the library in the driver</code> to preload dynamic libraries.</p> <h2 id="analyze-the-result"> <a href="#analyze-the-result" class="anchor-heading" aria-labelledby="analyze-the-result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Analyze the result </h2> <p>We can get [...]
diff --git a/output/archives/v1.1.1/developers/UsingJemallocWithCH.html b/output/archives/v1.1.1/developers/UsingJemallocWithCH.html
deleted file mode 100644
index a92de3e..0000000
--- a/output/archives/v1.1.1/developers/UsingJemallocWithCH.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } </style> <script src="/assets/js/vendor/lunr.min.js"></script> <script src="/assets/js/just-the-docs.js"></script> <meta name= [...]
- # At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracking.
- #
- # ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1).
--target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1)
-+#target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1)
-+target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
- target_link_libraries (_jemalloc PRIVATE unwind)
-</code></pre></div></div> <p>For more infomation, check https://github.com/jemalloc/jemalloc/issues/2282.</p> <h2 id="get-jeprof"> <a href="#get-jeprof" class="anchor-heading" aria-labelledby="get-jeprof"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Get jeprof </h2> <p>Change to the directory where you want to install jemalloc, and run the following commands:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre clas [...]
-</code></pre></div></div> <p>Then we get jeprof in the directory <code class="language-plaintext highlighter-rouge">$Clickhouse_SOURCE_PATH/contrib/jemalloc/bin/jeprof</code>.</p> <h2 id="compiler-libchso"> <a href="#compiler-libchso" class="anchor-heading" aria-labelledby="compiler-libchso"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Compiler libch.so </h2> <p>Ensure to enable jemalloc <code class="language-plaintext highlighter-rouge">-DENAB [...]
-</code></pre></div></div> <p>You can find more options in https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling.</p> <h2 id="analyze-the-result"> <a href="#analyze-the-result" class="anchor-heading" aria-labelledby="analyze-the-result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Analyze the result </h2> <p>After you get the heap file, you can use the following command to analyze the result: Check memory diff, so you can find whi [...]
-</code></pre></div></div> <p>You can find more usage about jeprof with <code class="language-plaintext highlighter-rouge">jeprof --help</code>.</p> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, [...]
diff --git a/output/archives/v1.1.1/developers/cpp_code_style/index.html b/output/archives/v1.1.1/developers/cpp_code_style/index.html
index bda7478..479973a 100644
--- a/output/archives/v1.1.1/developers/cpp_code_style/index.html
+++ b/output/archives/v1.1.1/developers/cpp_code_style/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/developers/docker_centos7/index.html b/output/archives/v1.1.1/developers/docker_centos7/index.html
index d0ed4c3..576fed6 100644
--- a/output/archives/v1.1.1/developers/docker_centos7/index.html
+++ b/output/archives/v1.1.1/developers/docker_centos7/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run -itd --name gluten centos:7 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum -y install epel-release centos-release-scl
diff --git a/output/archives/v1.1.1/developers/docker_centos8.html b/output/archives/v1.1.1/developers/docker_centos8.html
index 4016873..84c74e3 100644
--- a/output/archives/v1.1.1/developers/docker_centos8.html
+++ b/output/archives/v1.1.1/developers/docker_centos8.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run -itd --name gluten centos:8 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>#update mirror
diff --git a/output/archives/v1.1.1/developers/docker_ubuntu/index.html b/output/archives/v1.1.1/developers/docker_ubuntu/index.html
index 2891f69..a21fce1 100644
--- a/output/archives/v1.1.1/developers/docker_ubuntu/index.html
+++ b/output/archives/v1.1.1/developers/docker_ubuntu/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
diff --git a/output/archives/v1.1.1/developers/howto/index.html b/output/archives/v1.1.1/developers/howto/index.html
index f1de7e5..9d51d01 100644
--- a/output/archives/v1.1.1/developers/howto/index.html
+++ b/output/archives/v1.1.1/developers/howto/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-plaintext [...]
mvn clean package -Pspark-3.2 -Pbackends-velox -Prss
mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-velox -am -DtagsToInclude="io.glutenproject.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Darrow.version=11.0.0-gluten -Dexec.skip
@@ -13,7 +13,7 @@ gluten_home/backends-velox/generated-native-benchmark/
└── _SUCCESS
</code></pre></div> </div> </li> </ul> </li> <li>now, run benchmarks with GDB <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>cd gluten_home/cpp/build/velox/benchmarks/
gdb generic_benchmark
-</code></pre></div> </div> <ul> <li>When GDB load <code class="language-plaintext highlighter-rouge">generic_benchmark</code> successfully, you can set <code class="language-plaintext highlighter-rouge">breakpoint</code> on the <code class="language-plaintext highlighter-rouge">main</code> function with command <code class="language-plaintext highlighter-rouge">b main</code>, and then run with command <code class="language-plaintext highlighter-rouge">r</code>, then the process <code cla [...]
+</code></pre></div> </div> <ul> <li>When GDB load <code class="language-plaintext highlighter-rouge">generic_benchmark</code> successfully, you can set <code class="language-plaintext highlighter-rouge">breakpoint</code> on the <code class="language-plaintext highlighter-rouge">main</code> function with command <code class="language-plaintext highlighter-rouge">b main</code>, and then run with command <code class="language-plaintext highlighter-rouge">r</code>, then the process <code cla [...]
gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526'
</code></pre></div></div> <ul> <li>the <code class="language-plaintext highlighter-rouge">core-Executor task l-2000883-1671542526</code> represents the core file name.</li> </ul> <h1 id="how-to-run-tpc-h-on-velox-backend"> <a href="#how-to-run-tpc-h-on-velox-backend" class="anchor-heading" aria-labelledby="how-to-run-tpc-h-on-velox-backend"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> How to run TPC-H on Velox backend </h1> <p>Now, both Parquet [...]
diff --git a/output/archives/v1.1.1/developers/howtorelease/index.html b/output/archives/v1.1.1/developers/howtorelease/index.html
index bb8aca4..7e68921 100644
--- a/output/archives/v1.1.1/developers/howtorelease/index.html
+++ b/output/archives/v1.1.1/developers/howtorelease/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
$ gpg --full-generate-key
# list the GPG keys
diff --git a/output/archives/v1.1.1/developers/index.html b/output/archives/v1.1.1/developers/index.html
index 7583f7c..8de163a 100644
--- a/output/archives/v1.1.1/developers/index.html
+++ b/output/archives/v1.1.1/developers/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/developers/microbenchmark/index.html b/output/archives/v1.1.1/developers/microbenchmark/index.html
index 4b44683..b34ef0a 100644
--- a/output/archives/v1.1.1/developers/microbenchmark/index.html
+++ b/output/archives/v1.1.1/developers/microbenchmark/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3</span>
diff --git a/output/archives/v1.1.1/developers/newto/index.html b/output/archives/v1.1.1/developers/newto/index.html
index 15a37e2..ecf85e9 100644
--- a/output/archives/v1.1.1/developers/newto/index.html
+++ b/output/archives/v1.1.1/developers/newto/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/archives/v1.1.1/developers/substrait/index.html b/output/archives/v1.1.1/developers/substrait/index.html
index ee5a67e..ef68ce6 100644
--- a/output/archives/v1.1.1/developers/substrait/index.html
+++ b/output/archives/v1.1.1/developers/substrait/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/developers/velox_function_development/index.html b/output/archives/v1.1.1/developers/velox_function_development/index.html
index 9644ab0..cdc0cb0 100644
--- a/output/archives/v1.1.1/developers/velox_function_development/index.html
+++ b/output/archives/v1.1.1/developers/velox_function_development/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html b/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html
index 84b8834..48a59c8 100644
--- a/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html
+++ b/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <p>If you don’t care about development environment, you can skip this part.</p> <p>Otherwise, do:</p> <ol> <li>cl [...]
git clone --recursive --shallow-submodules -b clickhouse_backend https://github.com/Kyligence/ClickHouse.git
</code></pre></div> </div> </li> <li> <p>Configure cpp-ch ${GLUTEN_SOURCE}/cpp-ch can be treated as an add-on of Kyligence/Clickhouse</p> <p>First, initialize some configuration for this add-on:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="nb">export </span><span class="nv">GLUTEN_SOURCE</span><span class="o">=</span>/path/to/gluten
diff --git a/output/archives/v1.1.1/docs/clickhouse/index.html b/output/archives/v1.1.1/docs/clickhouse/index.html
index e84f689..4ee6831 100644
--- a/output/archives/v1.1.1/docs/clickhouse/index.html
+++ b/output/archives/v1.1.1/docs/clickhouse/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/index.html b/output/archives/v1.1.1/docs/index.html
index 6f2c23d..ad354b3 100644
--- a/output/archives/v1.1.1/docs/index.html
+++ b/output/archives/v1.1.1/docs/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/velox/build_parameters/index.html b/output/archives/v1.1.1/docs/velox/build_parameters/index.html
index b588c53..1f3c54f 100644
--- a/output/archives/v1.1.1/docs/velox/build_parameters/index.html
+++ b/output/archives/v1.1.1/docs/velox/build_parameters/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/velox/gcs/index.html b/output/archives/v1.1.1/docs/velox/gcs/index.html
index 8d3f274..168810f 100644
--- a/output/archives/v1.1.1/docs/velox/gcs/index.html
+++ b/output/archives/v1.1.1/docs/velox/gcs/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-sh highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage.roo [...]
</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.1.1/docs/velox/getting-started/index.html b/output/archives/v1.1.1/docs/velox/getting-started/index.html
index f89b047..042b82a 100644
--- a/output/archives/v1.1.1/docs/velox/getting-started/index.html
+++ b/output/archives/v1.1.1/docs/velox/getting-started/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
@@ -68,7 +68,7 @@ sudo chown <sparkuser>:<sparkuser> /var/lib/hadoop-hdfs/
</code></pre></div></div> <ul> <li>Specify the environment variable <a href="https://github.com/apache/hawq/blob/e9d43144f7e947e071bba48871af9da354d177d0/depends/libhdfs3/src/client/FileSystem.cpp#L56">KRB5CCNAME</a> and upload the kerberos ticket cache file</li> </ul> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>--conf spark.executorEnv.KRB5CCNAME=krb5cc_0000 --files /tmp/krb5cc_0000
</code></pre></div></div> <p>The ticket cache file can be found by <code class="language-plaintext highlighter-rouge">klist</code>.</p> <h2 id="aws-s3-support"> <a href="#aws-s3-support" class="anchor-heading" aria-labelledby="aws-s3-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> AWS S3 support </h2> <p>Velox supports S3 with the open source <a href="https://github.com/aws/aws-sdk-cpp">AWS C++ SDK</a> and Gluten uses Velox S3 connector t [...]
./dev/buildbundle-veloxbe.sh --enable_s3=ON
-</code></pre></div></div> <p>Currently there are several ways to asscess S3 in Spark. Please refer <a href="/archives/v1.1.1/docs/velox/s3/">Velox S3</a> part for more detailed configurations</p> <h2 id="celeborn-support"> <a href="#celeborn-support" class="anchor-heading" aria-labelledby="celeborn-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Celeborn support </h2> <p>Gluten with velox backend supports <a href="https://github.com/apach [...]
+</code></pre></div></div> <p>Currently there are several ways to asscess S3 in Spark. Please refer <a href="VeloxS3.md">Velox S3</a> part for more detailed configurations</p> <h2 id="celeborn-support"> <a href="#celeborn-support" class="anchor-heading" aria-labelledby="celeborn-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Celeborn support </h2> <p>Gluten with velox backend supports <a href="https://github.com/apache/incubator-celeborn" [...]
</code></pre></div></div> <p>Then add the Gluten and Spark Celeborn Client packages to your Spark application’s classpath(usually add them into <code class="language-plaintext highlighter-rouge">$SPARK_HOME/jars</code>).</p> <ul> <li>Celeborn: celeborn-client-spark-3-shaded_2.12-0.3.0-incubating.jar</li> <li>Gluten: gluten-velox-bundle-spark3.x_2.12-xx_xx_xx-SNAPSHOT.jar, gluten-thirdparty-lib-xx-xx.jar</li> </ul> <p>Currently to use Gluten following configurations are required in <code [...]
# celeborn master
diff --git a/output/archives/v1.1.1/docs/velox/index.html b/output/archives/v1.1.1/docs/velox/index.html
index 2245196..21dc321 100644
--- a/output/archives/v1.1.1/docs/velox/index.html
+++ b/output/archives/v1.1.1/docs/velox/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/velox/s3/index.html b/output/archives/v1.1.1/docs/velox/s3/index.html
index 173b254..ca52dc4 100644
--- a/output/archives/v1.1.1/docs/velox/s3/index.html
+++ b/output/archives/v1.1.1/docs/velox/s3/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
diff --git a/output/archives/v1.1.1/index.html b/output/archives/v1.1.1/index.html
index 04b13ee..5da106e 100644
--- a/output/archives/v1.1.1/index.html
+++ b/output/archives/v1.1.1/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul.nav-list:first-child > li > ul > li [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(1)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.2.1/docs/configuration/index.html b/output/archives/v1.2.1/docs/configuration/index.html
index 36635a1..8893835 100644
--- a/output/archives/v1.2.1/docs/configuration/index.html
+++ b/output/archives/v1.2.1/docs/configuration/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nv">sparkContext</span><span class="o">.</span><span class="py">setLocalProperty</span><span class="o">(</span><span class="n">key</span><span class="o">,</span> <span class="n">value</span><span class="o">)</span>
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"select * from demo_tables"</span><span class="o">).</span><span class="py">show</span><span class="o">()</span>
</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.2.1/docs/developers/UsingGperftoolsInCH.html b/output/archives/v1.2.1/docs/developers/UsingGperftoolsInCH.html
deleted file mode 100644
index ef8fb10..0000000
--- a/output/archives/v1.2.1/docs/developers/UsingGperftoolsInCH.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } </style> <script src="/assets/js/vendor/lunr.min.js"></script> <script src="/assets/js/just-the-docs.js"></script> <meta name= [...]
-export LD_PRELOAD=$tcmalloc_path,libch.so # load the library in the driver
---files $tcmalloc_path # upload the library to the cluster
---conf spark.executorEnv.LD_PRELOAD=./libtcmalloc_and_profiler.so,libch.so # load the library in the executor
---conf spark.executorEnv.HEAPPROFILE=/tmp/gluten_heap_perf # set the heap profile path, you can change to CPUPROFILE for CPU profiling
-</code></pre></div></div> <p>For thrift server on local machine, note using <code class="language-plaintext highlighter-rouge">export LD_PRELOAD="$tcmalloc_path libch.so" # load the library in the driver</code> to preload dynamic libraries.</p> <h2 id="analyze-the-result"> <a href="#analyze-the-result" class="anchor-heading" aria-labelledby="analyze-the-result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Analyze the result </h2> <p>We can get [...]
diff --git a/output/archives/v1.2.1/docs/developers/UsingJemallocWithCH.html b/output/archives/v1.2.1/docs/developers/UsingJemallocWithCH.html
deleted file mode 100644
index f59a389..0000000
--- a/output/archives/v1.2.1/docs/developers/UsingJemallocWithCH.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } </style> <script src="/assets/js/vendor/lunr.min.js"></script> <script src="/assets/js/just-the-docs.js"></script> <meta name= [...]
- # At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracking.
- #
- # ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1).
--target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1)
-+#target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1)
-+target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
- target_link_libraries (_jemalloc PRIVATE unwind)
-</code></pre></div></div> <p>For more infomation, check https://github.com/jemalloc/jemalloc/issues/2282.</p> <h2 id="get-jeprof"> <a href="#get-jeprof" class="anchor-heading" aria-labelledby="get-jeprof"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Get jeprof </h2> <p>Change to the directory where you want to install jemalloc, and run the following commands:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre clas [...]
-</code></pre></div></div> <p>Then we get jeprof in the directory <code class="language-plaintext highlighter-rouge">$Clickhouse_SOURCE_PATH/contrib/jemalloc/bin/jeprof</code>.</p> <h2 id="compiler-libchso"> <a href="#compiler-libchso" class="anchor-heading" aria-labelledby="compiler-libchso"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Compiler libch.so </h2> <p>Ensure to enable jemalloc <code class="language-plaintext highlighter-rouge">-DENAB [...]
-</code></pre></div></div> <p>You can find more options in https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling.</p> <h2 id="analyze-the-result"> <a href="#analyze-the-result" class="anchor-heading" aria-labelledby="analyze-the-result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Analyze the result </h2> <p>After you get the heap file, you can use the following command to analyze the result: Check memory diff, so you can find whi [...]
-</code></pre></div></div> <p>You can find more usage about jeprof with <code class="language-plaintext highlighter-rouge">jeprof --help</code>.</p> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, [...]
diff --git a/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html b/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html
index 9749167..85de550 100644
--- a/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html
+++ b/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/developers/docker-centos7/index.html b/output/archives/v1.2.1/docs/developers/docker-centos7/index.html
index 086fa45..19cc3a6 100644
--- a/output/archives/v1.2.1/docs/developers/docker-centos7/index.html
+++ b/output/archives/v1.2.1/docs/developers/docker-centos7/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:7 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum <span class="nt">-y</span> <span class="nb">install </span>epel-release centos-release-scl
diff --git a/output/archives/v1.2.1/docs/developers/docker-centos8/index.html b/output/archives/v1.2.1/docs/developers/docker-centos8/index.html
index 617ed23..12ef200 100644
--- a/output/archives/v1.2.1/docs/developers/docker-centos8/index.html
+++ b/output/archives/v1.2.1/docs/developers/docker-centos8/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:8 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">#update mirror</span>
diff --git a/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html b/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html
index 3bf2c7d..b8efa53 100644
--- a/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html
+++ b/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--network</span> host <span class="nt">--name</span> gluten ubuntu:22.04 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
diff --git a/output/archives/v1.2.1/docs/developers/how-to-release/index.html b/output/archives/v1.2.1/docs/developers/how-to-release/index.html
index bf9e4f6..016d103 100644
--- a/output/archives/v1.2.1/docs/developers/how-to-release/index.html
+++ b/output/archives/v1.2.1/docs/developers/how-to-release/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nv">$ </span>gpg <span class="nt">--full-generate-key</span>
<span class="c"># list the GPG keys</span>
diff --git a/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html b/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html
index 3713486..565225e 100644
--- a/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html
+++ b/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <ol> <li>Open the report file and check if there is any vulnerability issue highlighted. We must guarantee all the vulnerability issue has been solved before an official release.</li> </ol> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Founda [...]
diff --git a/output/archives/v1.2.1/docs/developers/how-to-use/index.html b/output/archives/v1.2.1/docs/developers/how-to-use/index.html
index 4f77d79..3943999 100644
--- a/output/archives/v1.2.1/docs/developers/how-to-use/index.html
+++ b/output/archives/v1.2.1/docs/developers/how-to-use/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-bash highl [...]
mvn clean package <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span>
mvn <span class="nb">test</span> <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span> <span class="nt">-pl</span> backends-velox <span class="nt">-am</span> <span class="nt">-DtagsToInclude</span><span class="o">=</span><span class="s2">"io.glutenproject.tags.GenerateExample"</span> <span class="nt">-Dtest</span><span class="o">=</span>none <span class="nt">-DfailIfNoTests</span><span class="o">=</span><span class="nb">false</span> <s [...]
diff --git a/output/archives/v1.2.1/docs/developers/index.html b/output/archives/v1.2.1/docs/developers/index.html
index c3f7f0d..af4f9c2 100644
--- a/output/archives/v1.2.1/docs/developers/index.html
+++ b/output/archives/v1.2.1/docs/developers/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html b/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html
index 530f1d0..397d335 100644
--- a/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html
+++ b/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3</span>
diff --git a/output/archives/v1.2.1/docs/developers/new-to/index.html b/output/archives/v1.2.1/docs/developers/new-to/index.html
index 6b81cb8..1f5aedd 100644
--- a/output/archives/v1.2.1/docs/developers/new-to/index.html
+++ b/output/archives/v1.2.1/docs/developers/new-to/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/archives/v1.2.1/docs/developers/substrait/index.html b/output/archives/v1.2.1/docs/developers/substrait/index.html
index 5c951bf..569f32f 100644
--- a/output/archives/v1.2.1/docs/developers/substrait/index.html
+++ b/output/archives/v1.2.1/docs/developers/substrait/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html b/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html
index 40e795a..a44e8f5 100644
--- a/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html
+++ b/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/archives/v1.2.1/docs/getting-started/abfs/index.html b/output/archives/v1.2.1/docs/getting-started/abfs/index.html
index f1c72f5..365421c 100644
--- a/output/archives/v1.2.1/docs/getting-started/abfs/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/abfs/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
-</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/build-guide/index.html b/output/archives/v1.2.1/docs/getting-started/build-guide/index.html
index 124ad82..abc8981 100644
--- a/output/archives/v1.2.1/docs/getting-started/build-guide/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/build-guide/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html b/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html
index 9307bb9..d2d3a0a 100644
--- a/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <p>If you don’t care about development environment, you can skip this part.</p> <p>Otherwise, do:</p> <ol> <li>cl [...]
git clone <span class="nt">--recursive</span> <span class="nt">--shallow-submodules</span> <span class="nt">-b</span> clickhouse_backend https://github.com/Kyligence/ClickHouse.git
</code></pre></div> </div> </li> <li> <p>Configure cpp-ch ${GLUTEN_SOURCE}/cpp-ch can be treated as an add-on of Kyligence/Clickhouse</p> <p>First, initialize some configuration for this add-on:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="nb">export </span><span class="nv">GLUTEN_SOURCE</span><span class="o">=</span>/path/to/gluten
diff --git a/output/archives/v1.2.1/docs/getting-started/gcs/index.html b/output/archives/v1.2.1/docs/getting-started/gcs/index.html
index b0f4469..dd18fa8 100644
--- a/output/archives/v1.2.1/docs/getting-started/gcs/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/gcs/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage. [...]
</code></pre></div></div> <h2 id="configuring-gcs-max-retry-count"> <a href="#configuring-gcs-max-retry-count" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-count"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry count </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected.</p> <div class="language-shell highligh [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/index.html b/output/archives/v1.2.1/docs/getting-started/index.html
index 920a6e2..5f74d59 100644
--- a/output/archives/v1.2.1/docs/getting-started/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/localcache/index.html b/output/archives/v1.2.1/docs/getting-started/localcache/index.html
index b5e41b2..47153da 100644
--- a/output/archives/v1.2.1/docs/getting-started/localcache/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/localcache/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.gluten.sql.columnar.backend.velox.memCacheSize // the total size of <span class="k">in</span><span class="nt">-mem</span> cache, default is 128MB.
spark.gluten.sql.columnar.backend.velox.ssdCachePath // the folder to store the cache files, default is <span class="s2">"/tmp"</span><span class="nb">.</span>
spark.gluten.sql.columnar.backend.velox.ssdCacheSize // the total size of the SSD cache, default is 128MB. Velox will <span class="k">do in</span><span class="nt">-mem</span> cache only <span class="k">if </span>this value is 0.
diff --git a/output/archives/v1.2.1/docs/getting-started/s3/index.html b/output/archives/v1.2.1/docs/getting-started/s3/index.html
index 77a47f4..7dc4e36 100644
--- a/output/archives/v1.2.1/docs/getting-started/s3/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/s3/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
@@ -8,4 +8,4 @@ spark.hadoop.fs.s3a.connection.ssl.enabled <span class="nb">true
</code></pre></div></div> <h2 id="configuring-s3-instance-credentials"> <a href="#configuring-s3-instance-credentials" class="anchor-heading" aria-labelledby="configuring-s3-instance-credentials"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring S3 instance credentials </h2> <p>S3 also provides other methods for accessing, you can also use instance credentials by setting the following config</p> <div class="language-plaintext highlighter [...]
</code></pre></div></div> <p>Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation.</p> <h2 id="configuring-s3-iam-roles"> <a href="#configuring-s3-iam-roles" class="anchor-heading" aria-labelledby="configuring-s3-iam-roles"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring S3 IAM roles </h2> <p>You can also use iam role credentials by setting the following c [...]
spark.hadoop.fs.s3a.iam.role.session.name xxxx
-</code></pre></div></div> <p>Note that <code class="language-plaintext highlighter-rouge">spark.hadoop.fs.s3a.iam.role.session.name</code> is optional.</p> <h2 id="other-authentatication-methods-are-not-supported-yet"> <a href="#other-authentatication-methods-are-not-supported-yet" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-supported-yet"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication met [...]
+</code></pre></div></div> <p>Note that <code class="language-plaintext highlighter-rouge">spark.hadoop.fs.s3a.iam.role.session.name</code> is optional.</p> <h2 id="other-authentatication-methods-are-not-supported-yet"> <a href="#other-authentatication-methods-are-not-supported-yet" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-supported-yet"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication met [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html b/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html
index d6e5cd6..d9559e9 100644
--- a/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
@@ -8,7 +8,7 @@
<span class="c">## fetch gluten code</span>
git clone https://github.com/apache/incubator-gluten.git
-</code></pre></div></div> <h1 id="build-gluten-with-velox-backend"> <a href="#build-gluten-with-velox-backend" class="anchor-heading" aria-labelledby="build-gluten-with-velox-backend"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Build Gluten with Velox Backend </h1> <p>It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. <a href="/archives/v1.2.1/docs/getting-started/build-guide/">Gluten build guide</a> listed the param [...]
+</code></pre></div></div> <h1 id="build-gluten-with-velox-backend"> <a href="#build-gluten-with-velox-backend" class="anchor-heading" aria-labelledby="build-gluten-with-velox-backend"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Build Gluten with Velox Backend </h1> <p>It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. <a href="build-guide.md">Gluten build guide</a> listed the parameters and their default value of bui [...]
</code></pre></div></div> <p>After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and setting up build dependencies.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./dev/buildbundle-veloxbe.sh <span class="nt">--enable_ep_cache</span><span class="o">=</span>ON <span class="nt">--build_arrow</span><span class="o">=</span>OFF <span class="nt">--run_setup_script</span [...]
</code></pre></div></div> <p><strong>For aarch64 build:</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span><span class="nv">CPU_TARGET</span><span class="o">=</span><span class="s2">"aarch64"</span>
@@ -57,9 +57,9 @@ mvn clean package <span class="nt">-Pbackends-velox</span> <span class="nt">-Pce
</code></pre></div></div> <ul> <li>Specify the environment variable <a href="https://github.com/apache/hawq/blob/e9d43144f7e947e071bba48871af9da354d177d0/depends/libhdfs3/src/client/FileSystem.cpp#L56">KRB5CCNAME</a> and upload the kerberos ticket cache file</li> </ul> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">--conf</span> spark.executorEnv.KRB5CCNAME<span class="o">=</span>krb5cc_0000 <span class="nt">--files</sp [...]
</code></pre></div></div> <p>The ticket cache file can be found by <code class="language-plaintext highlighter-rouge">klist</code>.</p> <h2 id="azure-blob-file-system-abfs-support"> <a href="#azure-blob-file-system-abfs-support" class="anchor-heading" aria-labelledby="azure-blob-file-system-abfs-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Azure Blob File System (ABFS) support </h2> <p>Velox supports ABFS with the open source <a href=" [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--enable_abfs</span><span class="o">=</span>ON
-</code></pre></div></div> <p>Please refer <a href="/archives/v1.2.1/docs/getting-started/abfs/">Velox ABFS</a> part for more detailed configurations.</p> <h2 id="aws-s3-support"> <a href="#aws-s3-support" class="anchor-heading" aria-labelledby="aws-s3-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> AWS S3 support </h2> <p>Velox supports S3 with the open source <a href="https://github.com/aws/aws-sdk-cpp">AWS C++ SDK</a> and Gluten uses Ve [...]
+</code></pre></div></div> <p>Please refer <a href="VeloxABFS.md">Velox ABFS</a> part for more detailed configurations.</p> <h2 id="aws-s3-support"> <a href="#aws-s3-support" class="anchor-heading" aria-labelledby="aws-s3-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> AWS S3 support </h2> <p>Velox supports S3 with the open source <a href="https://github.com/aws/aws-sdk-cpp">AWS C++ SDK</a> and Gluten uses Velox S3 connector to connect wit [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--enable_s3</span><span class="o">=</span>ON
-</code></pre></div></div> <p>Currently there are several ways to asscess S3 in Spark. Please refer <a href="/archives/v1.2.1/docs/getting-started/s3/">Velox S3</a> part for more detailed configurations</p> <h2 id="celeborn-support"> <a href="#celeborn-support" class="anchor-heading" aria-labelledby="celeborn-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Celeborn support </h2> <p>Gluten with velox backend supports <a href="https://github [...]
+</code></pre></div></div> <p>Currently there are several ways to asscess S3 in Spark. Please refer <a href="VeloxS3.md">Velox S3</a> part for more detailed configurations</p> <h2 id="celeborn-support"> <a href="#celeborn-support" class="anchor-heading" aria-labelledby="celeborn-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Celeborn support </h2> <p>Gluten with velox backend supports <a href="https://github.com/apache/celeborn">Celeborn< [...]
</code></pre></div></div> <p>Then add the Gluten and Spark Celeborn Client packages to your Spark application’s classpath(usually add them into <code class="language-plaintext highlighter-rouge">$SPARK_HOME/jars</code>).</p> <ul> <li>Celeborn: celeborn-client-spark-3-shaded_2.12-[celebornVersion].jar</li> <li>Gluten: gluten-velox-bundle-spark3.x_2.12-xx_xx_xx-SNAPSHOT.jar, gluten-celeborn-package-xx-SNAPSHOT.jar</li> </ul> <p>Currently to use Gluten following configurations are required [...]
<span class="c"># celeborn master</span>
@@ -226,7 +226,7 @@ var gluten_root = "/PATH/TO/GLUTEN"
<span class="nt">--executor-memory</span> 25g <span class="se">\</span>
<span class="nt">--conf</span> spark.executor.memoryOverhead<span class="o">=</span>5g <span class="se">\</span>
<span class="nt">--conf</span> spark.driver.maxResultSize<span class="o">=</span>32g
-</code></pre></div></div> <p>Refer to <a href="/archives/v1.2.1/docs/configuration/">Gluten configuration</a> for more details.</p> <h2 id="result"> <a href="#result" class="anchor-heading" aria-labelledby="result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Result </h2> <p><em>wholestagetransformer</em> indicates that the offloading works.</p> <p><img src="../image/TPC-H_Q6_DAG.png" alt="TPC-H Q6" /></p> <h2 id="performance"> <a href="#perfor [...]
+</code></pre></div></div> <p>Refer to <a href="../Configuration.md">Gluten configuration</a> for more details.</p> <h2 id="result"> <a href="#result" class="anchor-heading" aria-labelledby="result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Result </h2> <p><em>wholestagetransformer</em> indicates that the offloading works.</p> <p><img src="../image/TPC-H_Q6_DAG.png" alt="TPC-H Q6" /></p> <h2 id="performance"> <a href="#performance" class="anc [...]
Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L]
Arguments: false
Native Plan:
diff --git a/output/archives/v1.2.1/docs/index.html b/output/archives/v1.2.1/docs/index.html
index 9871e2d..69c951a 100644
--- a/output/archives/v1.2.1/docs/index.html
+++ b/output/archives/v1.2.1/docs/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/index.html b/output/archives/v1.2.1/docs/velox-backend/index.html
index 132f041..41fc0ad 100644
--- a/output/archives/v1.2.1/docs/velox-backend/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/limitations/index.html b/output/archives/v1.2.1/docs/velox-backend/limitations/index.html
index 10f7f3d..3d95c89 100644
--- a/output/archives/v1.2.1/docs/velox-backend/limitations/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/limitations/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="filesource-format"> <a href="#filesource-format" class="anchor-heading" aria-labelledby="filesource-format"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> FileSource format </h4> <p>Currently, Gluten only fully supports parquet file format and partially support ORC. If other format is used, scan operator falls back to vanilla spark.</p> <h4 id="partitioned-table-scan"> <a href="#partitioned-table-scan" class="anc [...]
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e"</span><span class="o">)</span>
</code></pre></div></div> <p>Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><spa [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/support/index.html b/output/archives/v1.2.1/docs/velox-backend/support/index.html
index 4f9604c..a4b17a5 100644
--- a/output/archives/v1.2.1/docs/velox-backend/support/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/support/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html b/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html
index 3335612..161aee0 100644
--- a/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
# velox backend only
rm -rf $SPARK_HOME/jars/flatbuffers-*
rm -rf $SPARK_HOME/jars/arrow-*
diff --git a/output/archives/v1.2.1/docs/velox-backend/udf/index.html b/output/archives/v1.2.1/docs/velox-backend/udf/index.html
index a2db93a..5a3583e 100644
--- a/output/archives/v1.2.1/docs/velox-backend/udf/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/udf/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li > a, .sit [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
#include <velox/expression/VectorFunction.h>
#include <velox/udf/Udf.h>
diff --git a/output/archives/v1.2.1/index.html b/output/archives/v1.2.1/index.html
index c16e341..bee1e74 100644
--- a/output/archives/v1.2.1/index.html
+++ b/output/archives/v1.2.1/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li:not(:nth-child(1)) > a, .site-nav > ul.nav-list:first-child > li > ul > li [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.3.0/configuration.html b/output/archives/v1.3.0/configuration.html
new file mode 100644
index 0000000..c0092f4
--- /dev/null
+++ b/output/archives/v1.3.0/configuration.html
@@ -0,0 +1,4 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<span class="nv">sparkContext</span><span class="o">.</span><span class="py">setLocalProperty</span><span class="o">(</span><span class="n">key</span><span class="o">,</span> <span class="n">value</span><span class="o">)</span>
+<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"select * from demo_tables"</span><span class="o">).</span><span class="py">show</span><span class="o">()</span>
+</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.3.0/developers.html b/output/archives/v1.3.0/developers.html
new file mode 100644
index 0000000..4700822
--- /dev/null
+++ b/output/archives/v1.3.0/developers.html
@@ -0,0 +1 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/developers/cpp-code-style/index.html b/output/archives/v1.3.0/developers/cpp-code-style/index.html
new file mode 100644
index 0000000..db27f02
--- /dev/null
+++ b/output/archives/v1.3.0/developers/cpp-code-style/index.html
@@ -0,0 +1 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/developers/docker-centos7/index.html b/output/archives/v1.3.0/developers/docker-centos7/index.html
new file mode 100644
index 0000000..de49f10
--- /dev/null
+++ b/output/archives/v1.3.0/developers/docker-centos7/index.html
@@ -0,0 +1,47 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:7 /bin/bash
+docker attach gluten
+</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum <span class="nt">-y</span> <span class="nb">install </span>epel-release centos-release-scl
+yum <span class="nt">-y</span> <span class="nb">install</span> <span class="se">\</span>
+ git236 <span class="se">\</span>
+ dnf <span class="se">\</span>
+ cmake3 <span class="se">\</span>
+ devtoolset-9 <span class="se">\</span>
+ java-1.8.0-openjdk <span class="se">\</span>
+ java-1.8.0-openjdk-devel <span class="se">\</span>
+ ninja-build <span class="se">\</span>
+ wget <span class="se">\</span>
+ autoconf <span class="se">\</span>
+ autoconf-archive <span class="se">\</span>
+ automake <span class="se">\</span>
+ perl-IPC-Cmd <span class="se">\</span>
+ libicu-devel <span class="se">\</span>
+ bison <span class="se">\</span>
+ libtool <span class="se">\</span>
+ patch <span class="se">\</span>
+ flex <span class="se">\</span>
+ <span class="nb">sudo</span>
+
+<span class="c"># gluten need maven version >=3.6.3</span>
+wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+<span class="nb">tar</span> <span class="nt">-xvf</span> apache-maven-3.8.8-bin.tar.gz
+<span class="nb">mv </span>apache-maven-3.8.8 /usr/lib/maven
+<span class="nb">export </span><span class="nv">MAVEN_HOME</span><span class="o">=</span>/usr/lib/maven
+<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="k">${</span><span class="nv">PATH</span><span class="k">}</span>:<span class="k">${</span><span class="nv">MAVEN_HOME</span><span class="k">}</span>/bin
+
+<span class="c"># cmake 3.x is required</span>
+<span class="nb">ln</span> <span class="nt">-s</span> /usr/bin/cmake3 /usr/local/bin/cmake
+
+<span class="c"># enable gcc 9</span>
+<span class="nb">.</span> /opt/rh/devtoolset-9/enable <span class="o">||</span> <span class="nb">exit </span>1
+
+<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-1.8.0-openjdk
+<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
+
+git clone https://github.com/oap-project/gluten.git
+<span class="nb">cd </span>gluten
+
+<span class="c"># To access HDFS or S3, you need to add the parameters `--enable_hdfs=ON` and `--enable_s3=ON`</span>
+<span class="c"># If you have the same error with issue-3283, you need to add the parameter `--compile_arrow_java=ON`</span>
+./dev/buildbundle-veloxbe.sh
+</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.3.0/developers/docker-centos8/index.html b/output/archives/v1.3.0/developers/docker-centos8/index.html
new file mode 100644
index 0000000..db7b9bd
--- /dev/null
+++ b/output/archives/v1.3.0/developers/docker-centos8/index.html
@@ -0,0 +1,35 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:8 /bin/bash
+docker attach gluten
+</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">#update mirror</span>
+<span class="nb">sed</span> <span class="nt">-i</span> <span class="nt">-e</span> <span class="s2">"s|mirrorlist=|#mirrorlist=|g"</span> /etc/yum.repos.d/CentOS-<span class="k">*</span>
+<span class="nb">sed</span> <span class="nt">-i</span> <span class="nt">-e</span> <span class="s2">"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"</span> /etc/yum.repos.d/CentOS-<span class="k">*</span>
+
+dnf <span class="nb">install</span> <span class="nt">-y</span> epel-release <span class="nb">sudo
+</span>yum <span class="nb">install</span> <span class="nt">-y</span> dnf-plugins-core
+yum config-manager <span class="nt">--set-enabled</span> powertools
+dnf <span class="nt">--enablerepo</span><span class="o">=</span>powertools <span class="nb">install</span> <span class="nt">-y</span> ninja-build
+dnf <span class="nt">--enablerepo</span><span class="o">=</span>powertools <span class="nb">install</span> <span class="nt">-y</span> libdwarf-devel
+dnf <span class="nb">install</span> <span class="nt">-y</span> <span class="nt">--setopt</span><span class="o">=</span><span class="nv">install_weak_deps</span><span class="o">=</span>False ccache gcc-toolset-9 git wget which libevent-devel <span class="se">\</span>
+ openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel <span class="se">\</span>
+ curl-devel cmake libicu-devel
+
+<span class="nb">source</span> /opt/rh/gcc-toolset-9/enable <span class="o">||</span> <span class="nb">exit </span>1
+
+yum <span class="nb">install</span> <span class="nt">-y</span> java-1.8.0-openjdk-devel patch
+<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-1.8.0-openjdk
+<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
+
+<span class="c">#gluten need maven version >=3.6.3</span>
+wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+<span class="nb">tar</span> <span class="nt">-xvf</span> apache-maven-3.8.8-bin.tar.gz
+<span class="nb">mv </span>apache-maven-3.8.8 /usr/lib/maven
+<span class="nb">export </span><span class="nv">MAVEN_HOME</span><span class="o">=</span>/usr/lib/maven
+<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="k">${</span><span class="nv">PATH</span><span class="k">}</span>:<span class="k">${</span><span class="nv">MAVEN_HOME</span><span class="k">}</span>/bin
+
+git clone https://github.com/oap-project/gluten.git
+<span class="nb">cd </span>gluten
+
+<span class="c"># To access HDFS or S3, you need to add the parameters `--enable_hdfs=ON` and `--enable_s3=ON`</span>
+./dev/buildbundle-veloxbe.sh
+</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.3.0/developers/docker-ubuntu/index.html b/output/archives/v1.3.0/developers/docker-ubuntu/index.html
new file mode 100644
index 0000000..b052052
--- /dev/null
+++ b/output/archives/v1.3.0/developers/docker-ubuntu/index.html
@@ -0,0 +1,44 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+docker run <span class="nt">-itd</span> <span class="nt">--network</span> host <span class="nt">--name</span> gluten ubuntu:22.04 /bin/bash
+docker attach gluten
+</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
+
+<span class="c">#install gcc and libraries to build arrow</span>
+apt <span class="nb">install </span>software-properties-common
+apt <span class="nb">install </span>maven build-essential cmake libssl-dev libre2-dev libcurl4-openssl-dev clang lldb lld libz-dev git ninja-build uuid-dev autoconf-archive curl zip unzip <span class="nb">tar </span>pkg-config bison libtool flex vim
+
+<span class="c">#velox script needs sudo to install dependency libraries</span>
+apt <span class="nb">install sudo</span>
+
+<span class="c"># make sure jemalloc is uninstalled, jemalloc will be build in vcpkg, which conflicts with the default jemalloc in system</span>
+apt purge libjemalloc-dev libjemalloc2 librust-jemalloc-sys-dev
+
+<span class="c">#make sure jdk8 is used. New version of jdk is not supported</span>
+apt <span class="nb">install</span> <span class="nt">-y</span> openjdk-8-jdk
+apt <span class="nb">install</span> <span class="nt">-y</span> default-jdk
+<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
+<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
+
+<span class="c">#manually install tzdata to avoid the interactive timezone config</span>
+<span class="nb">ln</span> <span class="nt">-fs</span> /usr/share/zoneinfo/America/New_York /etc/localtime
+<span class="nv">DEBIAN_FRONTEND</span><span class="o">=</span>noninteractive apt-get <span class="nb">install</span> <span class="nt">-y</span> tzdata
+dpkg <span class="nt">--configure</span> <span class="nt">-a</span>
+
+<span class="c">#setup proxy on necessary</span>
+<span class="c">#export http_proxy=xxxx</span>
+<span class="c">#export https_proxy=xxxx</span>
+
+<span class="c">#clone gluten</span>
+git clone https://github.com/oap-project/gluten.git
+<span class="nb">cd </span>gluten/
+
+<span class="c">#config maven proxy</span>
+<span class="c">#mkdir ~/.m2/</span>
+<span class="c">#vim ~/.m2/settings.xml</span>
+
+<span class="c"># the script download velox & arrow and compile all dependency library automatically</span>
+<span class="c"># To access HDFS or S3, you need to add the parameters `--enable_hdfs=ON` and `--enable_s3=ON`</span>
+<span class="c"># It's suggested to build using static link, enabled by `--enable_vcpkg=ON`</span>
+<span class="c"># For developer, it's suggested to enable Debug info, by --build_type=RelWithDebInfo. Note RelWithDebInfo uses -o2, release uses -o3</span>
+./dev/buildbundle-veloxbe.sh <span class="nt">--enable_vcpkg</span><span class="o">=</span>ON <span class="nt">--build_type</span><span class="o">=</span>RelWithDebInfo
+</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.3.0/developers/how-to-security-scan/index.html b/output/archives/v1.3.0/developers/how-to-security-scan/index.html
new file mode 100644
index 0000000..f71a98d
--- /dev/null
+++ b/output/archives/v1.3.0/developers/how-to-security-scan/index.html
@@ -0,0 +1,2 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+</code></pre></div></div> <ol> <li>Open the report file and check if there is any vulnerability issue highlighted. We must guarantee all the vulnerability issue has been solved before an official release.</li> </ol> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Founda [...]
diff --git a/output/archives/v1.3.0/developers/how-to-use/index.html b/output/archives/v1.3.0/developers/how-to-use/index.html
new file mode 100644
index 0000000..8031c82
--- /dev/null
+++ b/output/archives/v1.3.0/developers/how-to-use/index.html
@@ -0,0 +1,22 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-bash highl [...]
+mvn clean package <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span>
+mvn <span class="nb">test</span> <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span> <span class="nt">-pl</span> backends-velox <span class="nt">-am</span> <span class="nt">-DtagsToInclude</span><span class="o">=</span><span class="s2">"io.glutenproject.tags.GenerateExample"</span> <span class="nt">-Dtest</span><span class="o">=</span>none <span class="nt">-DfailIfNoTests</span><span class="o">=</span><span class="nb">false</span> <s [...]
+</code></pre></div> </div> <ul> <li>After the above operations, the examples files are generated under <code class="language-plaintext highlighter-rouge">gluten_home/backends-velox</code></li> <li>You can check it by the command <code class="language-plaintext highlighter-rouge">tree gluten_home/backends-velox/generated-native-benchmark/</code></li> <li>You may replace <code class="language-plaintext highlighter-rouge">-Pspark-3.2</code> with <code class="language-plaintext highlighter-r [...]
+gluten_home/backends-velox/generated-native-benchmark/
+├── example.json
+├── example_lineitem
+│ ├── part-00000-3ec19189-d20e-4240-85ae-88631d46b612-c000.snappy.parquet
+│ └── _SUCCESS
+└── example_orders
+ ├── part-00000-1e66fb98-4dd6-47a6-8679-8625dbc437ee-c000.snappy.parquet
+ └── _SUCCESS
+</code></pre></div> </div> </li> </ul> </li> <li>now, run benchmarks with GDB <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">cd </span>gluten_home/cpp/build/velox/benchmarks/
+gdb generic_benchmark
+</code></pre></div> </div> <ul> <li>When GDB load <code class="language-plaintext highlighter-rouge">generic_benchmark</code> successfully, you can set <code class="language-plaintext highlighter-rouge">breakpoint</code> on the <code class="language-plaintext highlighter-rouge">main</code> function with command <code class="language-plaintext highlighter-rouge">b main</code>, and then run with command <code class="language-plaintext highlighter-rouge">r</code>, then the process <code cla [...]
+gdb gluten_home/cpp/build/releases/libgluten.so <span class="s1">'core-Executor task l-2000883-1671542526'</span>
+
+</code></pre></div></div> <ul> <li>the <code class="language-plaintext highlighter-rouge">core-Executor task l-2000883-1671542526</code> represents the core file name.</li> </ul> <h1 id="how-to-run-tpc-h-on-velox-backend"> <a href="#how-to-run-tpc-h-on-velox-backend" class="anchor-heading" aria-labelledby="how-to-run-tpc-h-on-velox-backend"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> How to run TPC-H on Velox backend </h1> <p>Now, both Parquet [...]
+</code></pre></div> </div> </li> <li>set <code class="language-plaintext highlighter-rouge">var gluten_root</code> to correct directory. If <code class="language-plaintext highlighter-rouge">gluten_home</code> is the directory of <code class="language-plaintext highlighter-rouge">/home/gluten</code>, then modify it as below <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="k">var</span> <span class="n">gluten_root</span> <span [...]
+</code></pre></div> </div> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> - Modify `gluten_home/backends-velox/workload/tpch/run_tpch/tpch_parquet.sh`.
+</code></pre></div> </div> </li> <li>Set <code class="language-plaintext highlighter-rouge">GLUTEN_JAR</code> correctly. Please refer to the section of <a href="http://gluten.apache.org/docs/getting-started/velox-backend/#build-gluten-with-velox-backend">Build Gluten with Velox Backend</a></li> <li>Set <code class="language-plaintext highlighter-rouge">SPARK_HOME</code> correctly.</li> <li>Set the memory configurations appropriately. - Execute <code class="language-plaintext highlighter- [...]
diff --git a/output/docs/developers/microbenchmarks/index.html b/output/archives/v1.3.0/developers/microbenchmarks/index.html
similarity index 51%
rename from output/docs/developers/microbenchmarks/index.html
rename to output/archives/v1.3.0/developers/microbenchmarks/index.html
index 567c0f4..26b6af5 100644
--- a/output/docs/developers/microbenchmarks/index.html
+++ b/output/archives/v1.3.0/developers/microbenchmarks/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3</span>
diff --git a/output/docs/developers/new-to/index.html b/output/archives/v1.3.0/developers/new-to/index.html
similarity index 56%
rename from output/docs/developers/new-to/index.html
rename to output/archives/v1.3.0/developers/new-to/index.html
index 6077a4e..e282faa 100644
--- a/output/docs/developers/new-to/index.html
+++ b/output/archives/v1.3.0/developers/new-to/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/archives/v1.3.0/developers/substrait/index.html b/output/archives/v1.3.0/developers/substrait/index.html
new file mode 100644
index 0000000..58c01c4
--- /dev/null
+++ b/output/archives/v1.3.0/developers/substrait/index.html
@@ -0,0 +1 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/developers/velox-function-dev/index.html b/output/archives/v1.3.0/developers/velox-function-dev/index.html
new file mode 100644
index 0000000..385dc68
--- /dev/null
+++ b/output/archives/v1.3.0/developers/velox-function-dev/index.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+struct BitwiseAndFunction {
+ template <typename TInput>
+ // For void return type, it indicates null result will never be obtained for non-null input.
+ // For bool return type, it indicates null result can be obtained for non-null input (false for null).
+ FOLLY_ALWAYS_INLINE void call(TInput& result, TInput a, TInput b) {
+ result = a & b;
+ }
+};
+</code></pre></div></div> <p>It is templated, as well as the <code class="language-plaintext highlighter-rouge">call</code> function, to allow multiple types. In the above impl., the result will be null for null input. Please use <code class="language-plaintext highlighter-rouge">callNullable</code> if you need different behavior for null input, e.g., get a non-null result for null input. Also see <code class="language-plaintext highlighter-rouge">callNullFree</code> in velox document. I [...]
+</code></pre></div></div> <p>Functions for complex types have similar implementations. See <code class="language-plaintext highlighter-rouge">ArrayAverageFunction</code> in <a href="https://github.com/facebookincubator/velox/blob/main/velox/functions/prestosql/ArrayFunctions.h">velox/functions/prestosql/ArrayFunctions.h</a>.</p> <h3 id="reference"> <a href="#reference" class="anchor-heading" aria-labelledby="reference"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-lin [...]
diff --git a/output/archives/v1.3.0/getting-started.html b/output/archives/v1.3.0/getting-started.html
new file mode 100644
index 0000000..0cc3b9f
--- /dev/null
+++ b/output/archives/v1.3.0/getting-started.html
@@ -0,0 +1 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/getting-started/abfs/index.html b/output/archives/v1.3.0/getting-started/abfs/index.html
new file mode 100644
index 0000000..3df12ee
--- /dev/null
+++ b/output/archives/v1.3.0/getting-started/abfs/index.html
@@ -0,0 +1,2 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
diff --git a/output/archives/v1.3.0/getting-started/build-guide/index.html b/output/archives/v1.3.0/getting-started/build-guide/index.html
new file mode 100644
index 0000000..469a4b9
--- /dev/null
+++ b/output/archives/v1.3.0/getting-started/build-guide/index.html
@@ -0,0 +1 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/docs/getting-started/clickhouse-backend/index.html b/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html
similarity index 67%
rename from output/docs/getting-started/clickhouse-backend/index.html
rename to output/archives/v1.3.0/getting-started/clickhouse-backend/index.html
index 3a63c75..a2a6524 100644
--- a/output/docs/getting-started/clickhouse-backend/index.html
+++ b/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <p>If you don’t care about development environment, you can skip this part.</p> <p>Otherwise, do:</p> <ol> <li>cl [...]
git clone <span class="nt">--recursive</span> <span class="nt">--shallow-submodules</span> <span class="nt">-b</span> clickhouse_backend https://github.com/Kyligence/ClickHouse.git
</code></pre></div> </div> </li> <li> <p>Configure cpp-ch ${GLUTEN_SOURCE}/cpp-ch can be treated as an add-on of Kyligence/Clickhouse</p> <p>First, initialize some configuration for this add-on:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="nb">export </span><span class="nv">GLUTEN_SOURCE</span><span class="o">=</span>/path/to/gluten
diff --git a/output/archives/v1.3.0/getting-started/gcs/index.html b/output/archives/v1.3.0/getting-started/gcs/index.html
new file mode 100644
index 0000000..76a381f
--- /dev/null
+++ b/output/archives/v1.3.0/getting-started/gcs/index.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
+</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage. [...]
+</code></pre></div></div> <h2 id="configuring-gcs-max-retry-count"> <a href="#configuring-gcs-max-retry-count" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-count"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry count </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected.</p> <div class="language-shell highligh [...]
+</code></pre></div></div> <h2 id="configuring-gcs-max-retry-time"> <a href="#configuring-gcs-max-retry-time" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-time"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry time </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration.</p> <div class="language-shell highlig [...]
+</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.3.0/getting-started/localcache/index.html b/output/archives/v1.3.0/getting-started/localcache/index.html
new file mode 100644
index 0000000..db562b3
--- /dev/null
+++ b/output/archives/v1.3.0/getting-started/localcache/index.html
@@ -0,0 +1,8 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+spark.gluten.sql.columnar.backend.velox.memCacheSize // the total size of <span class="k">in</span><span class="nt">-mem</span> cache, default is 128MB.
+spark.gluten.sql.columnar.backend.velox.ssdCachePath // the folder to store the cache files, default is <span class="s2">"/tmp"</span><span class="nb">.</span>
+spark.gluten.sql.columnar.backend.velox.ssdCacheSize // the total size of the SSD cache, default is 128MB. Velox will <span class="k">do in</span><span class="nt">-mem</span> cache only <span class="k">if </span>this value is 0.
+spark.gluten.sql.columnar.backend.velox.ssdCacheShards // the shards of the SSD cache, default is 1.
+spark.gluten.sql.columnar.backend.velox.ssdCacheIOThreads // the IO threads <span class="k">for </span>cache promoting, default is 1. Velox will try to <span class="k">do</span> <span class="s2">"read-ahead"</span> <span class="k">if </span>this value is bigger than 1
+spark.gluten.sql.columnar.backend.velox.ssdODirect // <span class="nb">enable </span>or disable O_DIRECT on cache write, default false.
+</code></pre></div></div> <p>It’s recommended to mount SSDs to the cache path to get the best performance of local caching. Cache files will be written to “spark.gluten.sql.columnar.backend.velox.cachePath”, with UUID based suffix, e.g. “/tmp/cache.13e8ab65-3af4-46ac-8d28-ff99b2a9ec9b0”. Gluten cannot reuse older caches for now, and the old cache files are left after Spark context shutdown.</p> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https:// [...]
diff --git a/output/archives/v1.3.0/getting-started/s3/index.html b/output/archives/v1.3.0/getting-started/s3/index.html
new file mode 100644
index 0000000..39aa102
--- /dev/null
+++ b/output/archives/v1.3.0/getting-started/s3/index.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
+spark.hadoop.fs.s3a.access.key XXXXXXXXX
+spark.hadoop.fs.s3a.secret.key XXXXXXXXX
+spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com
+spark.hadoop.fs.s3a.connection.ssl.enabled <span class="nb">true
+</span>spark.hadoop.fs.s3a.path.style.access <span class="nb">false</span>
+</code></pre></div></div> <h2 id="configuring-s3-instance-credentials"> <a href="#configuring-s3-instance-credentials" class="anchor-heading" aria-labelledby="configuring-s3-instance-credentials"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring S3 instance credentials </h2> <p>S3 also provides other methods for accessing, you can also use instance credentials by setting the following config</p> <div class="language-plaintext highlighter [...]
+</code></pre></div></div> <p>Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation.</p> <h2 id="configuring-s3-iam-roles"> <a href="#configuring-s3-iam-roles" class="anchor-heading" aria-labelledby="configuring-s3-iam-roles"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring S3 IAM roles </h2> <p>You can also use iam role credentials by setting the following c [...]
+spark.hadoop.fs.s3a.iam.role.session.name xxxx
+</code></pre></div></div> <p>Note that <code class="language-plaintext highlighter-rouge">spark.hadoop.fs.s3a.iam.role.session.name</code> is optional.</p> <h2 id="other-authentatication-methods-are-not-supported-yet"> <a href="#other-authentatication-methods-are-not-supported-yet" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-supported-yet"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication met [...]
diff --git a/output/docs/getting-started/velox-backend/index.html b/output/archives/v1.3.0/getting-started/velox-backend/index.html
similarity index 63%
rename from output/docs/getting-started/velox-backend/index.html
rename to output/archives/v1.3.0/getting-started/velox-backend/index.html
index 5ab872e..57e86e4 100644
--- a/output/docs/getting-started/velox-backend/index.html
+++ b/output/archives/v1.3.0/getting-started/velox-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
@@ -8,7 +8,7 @@
<span class="c">## fetch gluten code</span>
git clone https://github.com/apache/incubator-gluten.git
-</code></pre></div></div> <h1 id="build-gluten-with-velox-backend"> <a href="#build-gluten-with-velox-backend" class="anchor-heading" aria-labelledby="build-gluten-with-velox-backend"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Build Gluten with Velox Backend </h1> <p>It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. <a href="/docs/getting-started/build-guide/">Gluten build guide</a> listed the parameters and their [...]
+</code></pre></div></div> <h1 id="build-gluten-with-velox-backend"> <a href="#build-gluten-with-velox-backend" class="anchor-heading" aria-labelledby="build-gluten-with-velox-backend"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Build Gluten with Velox Backend </h1> <p>It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. <a href="build-guide.md">Gluten build guide</a> listed the parameters and their default value of bui [...]
</code></pre></div></div> <p>After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and setting up build dependencies.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./dev/buildbundle-veloxbe.sh <span class="nt">--enable_ep_cache</span><span class="o">=</span>ON <span class="nt">--build_arrow</span><span class="o">=</span>OFF <span class="nt">--run_setup_script</span [...]
</code></pre></div></div> <p><strong>For aarch64 build:</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span><span class="nv">CPU_TARGET</span><span class="o">=</span><span class="s2">"aarch64"</span>
@@ -57,9 +57,9 @@ mvn clean package <span class="nt">-Pbackends-velox</span> <span class="nt">-Pce
</code></pre></div></div> <ul> <li>Specify the environment variable <a href="https://github.com/apache/hawq/blob/e9d43144f7e947e071bba48871af9da354d177d0/depends/libhdfs3/src/client/FileSystem.cpp#L56">KRB5CCNAME</a> and upload the kerberos ticket cache file</li> </ul> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">--conf</span> spark.executorEnv.KRB5CCNAME<span class="o">=</span>krb5cc_0000 <span class="nt">--files</sp [...]
</code></pre></div></div> <p>The ticket cache file can be found by <code class="language-plaintext highlighter-rouge">klist</code>.</p> <h2 id="azure-blob-file-system-abfs-support"> <a href="#azure-blob-file-system-abfs-support" class="anchor-heading" aria-labelledby="azure-blob-file-system-abfs-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Azure Blob File System (ABFS) support </h2> <p>Velox supports ABFS with the open source <a href=" [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--enable_abfs</span><span class="o">=</span>ON
-</code></pre></div></div> <p>Please refer <a href="/docs/getting-started/abfs/">Velox ABFS</a> part for more detailed configurations.</p> <h2 id="aws-s3-support"> <a href="#aws-s3-support" class="anchor-heading" aria-labelledby="aws-s3-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> AWS S3 support </h2> <p>Velox supports S3 with the open source <a href="https://github.com/aws/aws-sdk-cpp">AWS C++ SDK</a> and Gluten uses Velox S3 connector [...]
+</code></pre></div></div> <p>Please refer <a href="VeloxABFS.md">Velox ABFS</a> part for more detailed configurations.</p> <h2 id="aws-s3-support"> <a href="#aws-s3-support" class="anchor-heading" aria-labelledby="aws-s3-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> AWS S3 support </h2> <p>Velox supports S3 with the open source <a href="https://github.com/aws/aws-sdk-cpp">AWS C++ SDK</a> and Gluten uses Velox S3 connector to connect wit [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--enable_s3</span><span class="o">=</span>ON
-</code></pre></div></div> <p>Currently there are several ways to asscess S3 in Spark. Please refer <a href="/docs/getting-started/s3/">Velox S3</a> part for more detailed configurations</p> <h2 id="celeborn-support"> <a href="#celeborn-support" class="anchor-heading" aria-labelledby="celeborn-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Celeborn support </h2> <p>Gluten with velox backend supports <a href="https://github.com/apache/cele [...]
+</code></pre></div></div> <p>Currently there are several ways to asscess S3 in Spark. Please refer <a href="VeloxS3.md">Velox S3</a> part for more detailed configurations</p> <h2 id="celeborn-support"> <a href="#celeborn-support" class="anchor-heading" aria-labelledby="celeborn-support"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Celeborn support </h2> <p>Gluten with velox backend supports <a href="https://github.com/apache/celeborn">Celeborn< [...]
</code></pre></div></div> <p>Then add the Gluten and Spark Celeborn Client packages to your Spark application’s classpath(usually add them into <code class="language-plaintext highlighter-rouge">$SPARK_HOME/jars</code>).</p> <ul> <li>Celeborn: celeborn-client-spark-3-shaded_2.12-[celebornVersion].jar</li> <li>Gluten: gluten-velox-bundle-spark3.x_2.12-xx_xx_xx-SNAPSHOT.jar, gluten-celeborn-package-xx-SNAPSHOT.jar</li> </ul> <p>Currently to use Gluten following configurations are required [...]
<span class="c"># celeborn master</span>
@@ -226,7 +226,7 @@ var gluten_root = "/PATH/TO/GLUTEN"
<span class="nt">--executor-memory</span> 25g <span class="se">\</span>
<span class="nt">--conf</span> spark.executor.memoryOverhead<span class="o">=</span>5g <span class="se">\</span>
<span class="nt">--conf</span> spark.driver.maxResultSize<span class="o">=</span>32g
-</code></pre></div></div> <p>Refer to <a href="/docs/configuration/">Gluten configuration</a> for more details.</p> <h2 id="result"> <a href="#result" class="anchor-heading" aria-labelledby="result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Result </h2> <p><em>wholestagetransformer</em> indicates that the offloading works.</p> <p><img src="../image/TPC-H_Q6_DAG.png" alt="TPC-H Q6" /></p> <h2 id="performance"> <a href="#performance" class="an [...]
+</code></pre></div></div> <p>Refer to <a href="../Configuration.md">Gluten configuration</a> for more details.</p> <h2 id="result"> <a href="#result" class="anchor-heading" aria-labelledby="result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Result </h2> <p><em>wholestagetransformer</em> indicates that the offloading works.</p> <p><img src="../image/TPC-H_Q6_DAG.png" alt="TPC-H Q6" /></p> <h2 id="performance"> <a href="#performance" class="anc [...]
Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L]
Arguments: false
Native Plan:
diff --git a/output/archives/v1.3.0/index.html b/output/archives/v1.3.0/index.html
new file mode 100644
index 0000000..f5d80f2
--- /dev/null
+++ b/output/archives/v1.3.0/index.html
@@ -0,0 +1 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(3)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.3.0/velox-backend.html b/output/archives/v1.3.0/velox-backend.html
new file mode 100644
index 0000000..227a5d8
--- /dev/null
+++ b/output/archives/v1.3.0/velox-backend.html
@@ -0,0 +1 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/velox-backend/limitations/index.html b/output/archives/v1.3.0/velox-backend/limitations/index.html
new file mode 100644
index 0000000..6d474e4
--- /dev/null
+++ b/output/archives/v1.3.0/velox-backend/limitations/index.html
@@ -0,0 +1,24 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+</code></pre></div></div> <h4 id="filesource-format"> <a href="#filesource-format" class="anchor-heading" aria-labelledby="filesource-format"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> FileSource format </h4> <p>Currently, Gluten only fully supports parquet file format and partially support ORC. If other format is used, scan operator falls back to vanilla spark.</p> <h4 id="partitioned-table-scan"> <a href="#partitioned-table-scan" class="anc [...]
+<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e"</span><span class="o">)</span>
+</code></pre></div></div> <p>Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><spa [...]
+ <span class="o">.</span><span class="py">write</span>
+ <span class="o">.</span><span class="py">format</span><span class="o">(</span><span class="s">"parquet"</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">partitionBy</span><span class="o">(</span><span class="s">"p"</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">save</span><span class="o">(</span><span class="nv">f</span><span class="o">.</span><span class="py">getCanonicalPath</span><span class="o">)</span>
+</code></pre></div></div> <h4 id="partition-write-for-spark34-and-later"> <a href="#partition-write-for-spark34-and-later" class="anchor-heading" aria-labelledby="partition-write-for-spark34-and-later"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Partition write (For Spark3.4 and later) </h4> <p>Gluten supports static partition writes and dynamic partition writes.</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre cla [...]
+<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d) SELECT 2 as d, 3 as e"</span><span class="o">)</span>
+</code></pre></div></div> <p>Gluten does not support bucket write, and will fall back to vanilla Spark.</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><span class="o">,</span> <span class= [...]
+ <span class="o">.</span><span class="py">write</span>
+ <span class="o">.</span><span class="py">format</span><span class="o">(</span><span class="s">"parquet"</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">bucketBy</span><span class="o">(</span><span class="mi">2</span><span class="o">,</span> <span class="s">"c1"</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">save</span><span class="o">(</span><span class="nv">f</span><span class="o">.</span><span class="py">getCanonicalPath</span><span class="o">)</span>
+</code></pre></div></div> <h4 id="ctas-write-for-spark32-and-spark33"> <a href="#ctas-write-for-spark32-and-spark33" class="anchor-heading" aria-labelledby="ctas-write-for-spark32-and-spark33"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> CTAS write (For Spark3.2 and Spark3.3) </h4> <p>Gluten does not create table as select. It may raise exception. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlig [...]
+ <span class="o">.</span><span class="py">write</span>
+ <span class="o">.</span><span class="py">format</span><span class="o">(</span><span class="s">"parquet"</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">saveAsTable</span><span class="o">(</span><span class="s">"velox_ctas"</span><span class="o">)</span>
+</code></pre></div></div> <h4 id="ctas-write-for-spark34-and-later"> <a href="#ctas-write-for-spark34-and-later" class="anchor-heading" aria-labelledby="ctas-write-for-spark34-and-later"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> CTAS write (For Spark3.4 and later) </h4> <p>Gluten supports create table as select with parquet file format.</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><spa [...]
+ <span class="o">.</span><span class="py">write</span>
+ <span class="o">.</span><span class="py">format</span><span class="o">(</span><span class="s">"parquet"</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">saveAsTable</span><span class="o">(</span><span class="s">"velox_ctas"</span><span class="o">)</span>
+</code></pre></div></div> <h4 id="hivefileformat-write"> <a href="#hivefileformat-write" class="anchor-heading" aria-labelledby="hivefileformat-write"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> HiveFileFormat write </h4> <p>Gluten supports writes of HiveFileFormat when the output file type is of type <code class="language-plaintext highlighter-rouge">parquet</code> only</p> <h4 id="nan-support"> <a href="#nan-support" class="anchor-heading" a [...]
diff --git a/output/archives/v1.3.0/velox-backend/support/index.html b/output/archives/v1.3.0/velox-backend/support/index.html
new file mode 100644
index 0000000..b11604c
--- /dev/null
+++ b/output/archives/v1.3.0/velox-backend/support/index.html
@@ -0,0 +1 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/velox-backend/troubleshooting/index.html b/output/archives/v1.3.0/velox-backend/troubleshooting/index.html
new file mode 100644
index 0000000..2a8aae3
--- /dev/null
+++ b/output/archives/v1.3.0/velox-backend/troubleshooting/index.html
@@ -0,0 +1,46 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+# velox backend only
+rm -rf $SPARK_HOME/jars/flatbuffers-*
+rm -rf $SPARK_HOME/jars/arrow-*
+</code></pre></div></div> <h3 id="incompatible-class-error-when-using-native-writer"> <a href="#incompatible-class-error-when-using-native-writer" class="anchor-heading" aria-labelledby="incompatible-class-error-when-using-native-writer"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Incompatible class error when using native writer </h3> <p>Gluten native writer overwrite some vanilla spark classes. Therefore, when running a program that uses glu [...]
+spark.files = {absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
+// The absolute path on running node
+spark.driver.extraClassPath={absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
+// The relative path under the executor working directory
+spark.executor.extraClassPath=./gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
+</code></pre></div></div> <h4 id="configurations-for-yarn-cluster-mode"> <a href="#configurations-for-yarn-cluster-mode" class="anchor-heading" aria-labelledby="configurations-for-yarn-cluster-mode"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configurations for Yarn Cluster mode </h4> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.driver.userClassPathFirst = true
+spark.executor.userClassPathFirst = true
+
+spark.files = {absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
+// The relative path under the executor working directory
+spark.driver.extraClassPath=./gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
+// The relative path under the executor working directory
+spark.executor.extraClassPath=./gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
+</code></pre></div></div> <h4 id="configurations-for-local--standalone-mode"> <a href="#configurations-for-local--standalone-mode" class="anchor-heading" aria-labelledby="configurations-for-local--standalone-mode"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configurations for Local & Standalone mode </h4> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>// The absolute path on running node
+spark.driver.extraClassPath={absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
+// The absolute path on running node
+spark.executor.extraClassPath={absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
+</code></pre></div></div> <h3 id="invalid-pointer-error"> <a href="#invalid-pointer-error" class="anchor-heading" aria-labelledby="invalid-pointer-error"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Invalid pointer error </h3> <p>If the below error is reported at runtime, please re-build gluten with <code class="language-plaintext highlighter-rouge">--compile_arrow_java=ON</code>, then redeploy Gluten jar.</p> <div class="language-plaintext hig [...]
+======= Backtrace: =========
+/lib64/libc.so.6(+0x7d1fd)[0x7f38c29da1fd]
+/lib64/libstdc++.so.6(_ZNSt6locale5_Impl16_M_install_facetEPKNS_2idEPKNS_5facetE+0x142)[0x7f36cb3370d2]
+/lib64/libstdc++.so.6(_ZNSt6locale5_ImplC1Em+0x1e3)[0x7f36cb337523]
+/lib64/libstdc++.so.6(+0x71495)[0x7f36cb338495]
+/lib64/libpthread.so.0(pthread_once+0x50)[0x7f38c3147be0]
+/lib64/libstdc++.so.6(+0x714e1)[0x7f36cb3384e1]
+/lib64/libstdc++.so.6(_ZNSt6localeC2Ev+0x13)[0x7f36cb338523]
+/lib64/libstdc++.so.6(_ZNSt8ios_base4InitC2Ev+0xbc)[0x7f36cb33537c]
+/tmp/jnilib-645156599284574767.tmp(+0x2a90)[0x7f375d235a90]
+/lib64/ld-linux-x86-64.so.2(+0xf4e3)[0x7f38c33664e3]
+/lib64/ld-linux-x86-64.so.2(+0x13b04)[0x7f38c336ab04]
+/lib64/ld-linux-x86-64.so.2(+0xf2f4)[0x7f38c33662f4]
+/lib64/ld-linux-x86-64.so.2(+0x1321b)[0x7f38c336a21b]
+/lib64/libdl.so.2(+0x102b)[0x7f38c2d1f02b]
+/lib64/ld-linux-x86-64.so.2(+0xf2f4)[0x7f38c33662f4]
+/lib64/libdl.so.2(+0x162d)[0x7f38c2d1f62d]
+/lib64/libdl.so.2(dlopen+0x31)[0x7f38c2d1f0c1]
+/usr/local/jdk1.8.0_381/jre/lib/amd64/server/libjvm.so(+0x9292b1)[0x7f38c22732b1]
+/usr/local/jdk1.8.0_381/jre/lib/amd64/server/libjvm.so(JVM_LoadLibrary+0xa1)[0x7f38c205e0c1]
+/usr/local/jdk1.8.0_381/jre/lib/amd64/libjava.so(Java_java_lang_ClassLoader_00024NativeLibrary_load+0x1ac)
+...
+</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.3.0/velox-backend/udf/index.html b/output/archives/v1.3.0/velox-backend/udf/index.html
new file mode 100644
index 0000000..200971c
--- /dev/null
+++ b/output/archives/v1.3.0/velox-backend/udf/index.html
@@ -0,0 +1,87 @@
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+
+#include <velox/expression/VectorFunction.h>
+#include <velox/udf/Udf.h>
+
+namespace {
+static const char* kInteger = "integer";
+}
+
+const int kNumMyUdf = 1;
+
+const char* myUdfArgs[] = {kInteger}:
+gluten::UdfEntry myUdfSig = {"myudf", kInteger, 1, myUdfArgs};
+
+class MyUdf : public facebook::velox::exec::VectorFunction {
+ ... // Omit concrete implementation
+}
+
+static std::vector<std::shared_ptr<exec::FunctionSignature>>
+myUdfSignatures() {
+ return {facebook::velox::exec::FunctionSignatureBuilder()
+ .returnType(myUdfSig.dataType)
+ .argumentType(myUdfSig.argTypes[0])
+ .build()};
+}
+
+DEFINE_GET_NUM_UDF { return kNumMyUdf; }
+
+DEFINE_GET_UDF_ENTRIES { udfEntries[0] = myUdfSig; }
+
+DEFINE_REGISTER_UDF {
+ facebook::velox::exec::registerVectorFunction(
+ myUdf[0].name, myUdfSignatures(), std::make_unique<MyUdf>());
+}
+
+</code></pre></div> </div> </li> </ul> <p>To build the UDF library, users need to compile the C++ code and link to <code class="language-plaintext highlighter-rouge">libvelox.so</code>. It’s recommended to create a CMakeLists.txt for the project. Here’s an example:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>project(myudf)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+set(GLUTEN_HOME /path/to/gluten)
+
+add_library(myudf SHARED "MyUDF.cpp")
+
+find_library(VELOX_LIBRARY REQUIRED NAMES velox HINTS ${GLUTEN_HOME}/cpp/build/releases NO_DEFAULT_PATH)
+
+target_include_directories(myudf PRIVATE ${GLUTEN_HOME}/cpp ${GLUTEN_HOME}/ep/build-velox/build/velox_ep)
+target_link_libraries(myudf PRIVATE ${VELOX_LIBRARY})
+</code></pre></div></div> <p>The steps for creating and building a UDAF library are quite similar to those for a UDF library. The major difference lies in including and defining specific functions within the UDAF header file <a href="../../cpp/velox/udf/Udaf.h">Udaf.h</a></p> <ul> <li><code class="language-plaintext highlighter-rouge">getNumUdaf()</code></li> <li><code class="language-plaintext highlighter-rouge">getUdafEntries(gluten::UdafEntry* udafEntries)</code></li> <li><code class= [...]
+<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>libmyudf.so
+<span class="c"># Needed for Yarn client mode</span>
+<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths<span class="o">=</span>file:///path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so
+</code></pre></div></div> <ul> <li>Use the <code class="language-plaintext highlighter-rouge">--archives</code> option to upload an archive and configure its relative path</li> </ul> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">--archives</span> /path/to/udf_archives.zip#udf_archives
+<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>udf_archives
+<span class="c"># Needed for Yarn client mode</span>
+<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths<span class="o">=</span>file:///path/to/udf_archives.zip
+</code></pre></div></div> <ul> <li>Configure URI</li> </ul> <p>You can also specify the local or HDFS URIs to the UDF libraries or archives. Local URIs should exist on driver and every worker nodes.</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>file:///path/to/library_or_archive
+</code></pre></div></div> <h2 id="try-the-example"> <a href="#try-the-example" class="anchor-heading" aria-labelledby="try-the-example"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Try the example </h2> <p>We provided Velox UDF examples in file <a href="../../cpp/velox/udf/examples/MyUDF.cc">MyUDF.cc</a> and UDAF examples in file <a href="../../cpp/velox/udf/examples/MyUDAF.cc">MyUDAF.cc</a>. You need to build the gluten project with <code clas [...]
+</code></pre></div></div> <p>Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/</p> <p>Start spark-shell or spark-sql with below configuration</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># Use the `--files` option to upload a library and configure its relative path</span>
+<span class="nt">--files</span> /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so
+<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>libmyudf.so
+</code></pre></div></div> <p>or</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># Only configure URI</span>
+<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>file:///path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so
+</code></pre></div></div> <p>Run query. The functions <code class="language-plaintext highlighter-rouge">myudf1</code> and <code class="language-plaintext highlighter-rouge">myudf2</code> increment the input value by a constant of 5</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>select myudf1(100L), myudf2(1)
+</code></pre></div></div> <p>The output from spark-shell will be like</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>+------------------+----------------+
+|udfexpression(100)|udfexpression(1)|
++------------------+----------------+
+| 105| 6|
++------------------+----------------+
+</code></pre></div></div> <h2 id="configurations"> <a href="#configurations" class="anchor-heading" aria-labelledby="configurations"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configurations </h2> <div class="table-wrapper"><table> <thead> <tr> <th>Parameters</th> <th>Description</th> </tr> </thead> <tbody> <tr> <td>spark.gluten.sql.columnar.backend.velox.udfLibraryPaths</td> <td>Path to the udf/udaf libraries.</td> </tr> <tr> <td>spark.glute [...]
+pip3 install pandas pyarrow
+</code></pre></div></div> <p>Gluten provides a config to control enable <code class="language-plaintext highlighter-rouge">ColumnarArrowEvalPython</code> or not, with <code class="language-plaintext highlighter-rouge">true</code> as defalt.</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.gluten.sql.columnar.arrowUdf
+</code></pre></div></div> <p>Then take following <code class="language-plaintext highlighter-rouge">PySpark</code> code for example:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>from pyspark.sql.functions import pandas_udf, PandasUDFType
+import pyspark.sql.functions as F
+import os
+@pandas_udf('long')
+def pandas_plus_one(v):
+ return (v + 1)
+df = spark.read.orc("path_to_file").select("quantity").withColumn("processed_quantity", pandas_plus_one("quantity")).select("quantity")
+</code></pre></div></div> <p>The expected physical plan will be:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>== Physical Plan ==
+VeloxColumnarToRowExec
++- ^(2) ProjectExecTransformer [pythonUDF0#45L AS processed_quantity#41L]
+ +- ^(2) InputIteratorTransformer[quantity#2L, pythonUDF0#45L]
+ +- ^(2) InputAdapter
+ +- ^(2) ColumnarArrowEvalPython [pandas_plus_one(quantity#2L)#40L], [pythonUDF0#45L], 200
+ +- ^(1) NativeFileScan orc [quantity#2L] Batched: true, DataFilters: [], Format: ORC, Location: InMemoryFileIndex(1 paths)[file:/***], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<quantity:bigint>
+</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/assets/js/search-data.json b/output/assets/js/search-data.json
index aa275bf..8d17f8a 100644
--- a/output/assets/js/search-data.json
+++ b/output/assets/js/search-data.json
@@ -1,3551 +1,3558 @@
{"0": {
- "doc": "Getting start with ClickHouse Backend(v1.2.1)",
- "title": "ClickHouse Backend",
- "content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
- "url": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/#clickhouse-backend",
+ "doc": "About",
+ "title": "About",
+ "content": "This is the base Jekyll theme. You can find out more info about customizing your Jekyll theme, as well as basic Jekyll usage documentation at jekyllrb.com . You can find the source code for Minima at GitHub: jekyll / minima . You can find the source code for Jekyll at GitHub: jekyll / jekyll . ",
+ "url": "/about/",
- "relUrl": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/#clickhouse-backend"
+ "relUrl": "/about/"
},"1": {
- "doc": "Getting start with ClickHouse Backend(v1.2.1)",
- "title": "Getting start with ClickHouse Backend(v1.2.1)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/",
+ "doc": "Blog",
+ "title": "Latest Posts",
+ "content": ". | ",
+ "url": "/blog/",
- "relUrl": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/"
+ "relUrl": "/blog/"
},"2": {
- "doc": "v1.1.1/ClickHouse Backend",
- "title": "Gluten Documents by version",
- "content": " ",
- "url": "/archives/v1.1.1/docs/clickhouse/#gluten-documents-by-version",
+ "doc": "Blog",
+ "title": "Gluten 1.2.0 Release",
+ "content": "On Sept. 3rd 2024, I am thrilled to share that Apache Gluten (incubating) has celebrated its 1st Apache official release, version 1.2.0, featuring the following highlights. | ",
+ "url": "/blog/",
- "relUrl": "/archives/v1.1.1/docs/clickhouse/#gluten-documents-by-version"
+ "relUrl": "/blog/"
},"3": {
- "doc": "v1.1.1/ClickHouse Backend",
- "title": "v1.1.1/ClickHouse Backend",
- "content": " ",
- "url": "/archives/v1.1.1/docs/clickhouse/",
+ "doc": "Blog",
+ "title": "Welcome to Jekyll!",
+ "content": "You’ll find this post in your _posts directory. Go ahead and edit it and re-build the site to see your changes. You can rebuild the site in many different ways, but the most common way is to run jekyll serve, which launches a web server and auto-regenerates your site when a file is updated. | . ",
+ "url": "/blog/",
- "relUrl": "/archives/v1.1.1/docs/clickhouse/"
+ "relUrl": "/blog/"
},"4": {
- "doc": "Getting start with ClickHouse Backend",
- "title": "ClickHouse Backend",
- "content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
- "url": "/docs/getting-started/clickhouse-backend/#clickhouse-backend",
+ "doc": "Blog",
+ "title": "Blog",
+ "content": " ",
+ "url": "/blog/",
- "relUrl": "/docs/getting-started/clickhouse-backend/#clickhouse-backend"
+ "relUrl": "/blog/"
},"5": {
- "doc": "Getting start with ClickHouse Backend",
- "title": "Getting start with ClickHouse Backend",
- "content": " ",
- "url": "/docs/getting-started/clickhouse-backend/",
+ "doc": "Contact Us",
+ "title": "Contact Us",
+ "content": "This plugin is still under active development now, and doesn’t have a stable release. Welcome to evaluate it. If you encounter any issues or have any suggestions, please submit to our issue list. We’d love to hear your feedback . Apache has meticulously set up dedicated mailing lists for every project, underscoring the pivotal role of these platforms in fostering communication within the Apache community. Mailing lists serve as the backbone for numerous aspects of communi [...]
+ "url": "/contact/",
- "relUrl": "/docs/getting-started/clickhouse-backend/"
+ "relUrl": "/contact/"
},"6": {
- "doc": "Configuration(v1.2.1)",
- "title": "Spark Configurations for Gluten Plugin",
- "content": "There are many configurations could impact the Gluten Plugin performance and can be fine-tuned in Spark. You can add these configurations into spark-defaults.conf to enable or disable the setting. ",
- "url": "/archives/v1.2.1/docs/configuration/#spark-configurations-for-gluten-plugin",
+ "doc": "Contact Us",
+ "title": "How to subscribe Gluten community",
+ "content": "To subscribe to the dev@gluten.apache.org mailing list as an example, please follow these steps: . | Send a blank email to dev-subscribe@gluten.apache.org | Check your inbox for an email titled confirm subscribe to dev@gluten.apache.org If not received within a reasonable time, ensure it’s not blocked by your email provider. If not blocked and no reply is received after a while, repeat step 1 | Reply directly to the confirmation email without altering the subject or addin [...]
+ "url": "/contact/#how-to-subscribe-gluten-community",
- "relUrl": "/archives/v1.2.1/docs/configuration/#spark-configurations-for-gluten-plugin"
+ "relUrl": "/contact/#how-to-subscribe-gluten-community"
},"7": {
- "doc": "Configuration(v1.2.1)",
- "title": "Spark parameters",
- "content": "| Parameters | Description | Recommend Setting | . | spark.driver.extraClassPath | To add Gluten Plugin jar file in Spark Driver | /path/to/jar_file | . | spark.executor.extraClassPath | To add Gluten Plugin jar file in Spark Executor | /path/to/jar_file | . | spark.executor.memory | To set up how much memory to be used for Spark Executor. | | . | spark.memory.offHeap.size | To set up how much memory to be used for Java OffHeap. Please notice Gluten Plugin will leverage [...]
- "url": "/archives/v1.2.1/docs/configuration/#spark-parameters",
+ "doc": "Contact Us",
+ "title": "Issues and Discussions",
+ "content": "We use github to track bugs, feature requests, and answer questions. File an issue for a bug or feature request. ",
+ "url": "/contact/#issues-and-discussions",
- "relUrl": "/archives/v1.2.1/docs/configuration/#spark-parameters"
+ "relUrl": "/contact/#issues-and-discussions"
},"8": {
- "doc": "Configuration(v1.2.1)",
- "title": "Velox Parameters",
- "content": "The following configurations are related to Velox settings. | Parameters | Description | Recommend Setting | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The m [...]
- "url": "/archives/v1.2.1/docs/configuration/#velox-parameters",
+ "doc": "Contributing to Gluten",
+ "title": "How to become a committer",
+ "content": "To initiate your contributions to Gluten, understand the contribution process—any individual can submit patches, documentation, and examples to the project. The PMC routinely incorporates fresh committers from the pool of active contributors, assessing their contributions to Gluten. The criteria for new committers encompass: . | The Scope of Contribution: To expand contributions beyond pull requests, individuals can engage in various activities including reviewing other p [...]
+ "url": "/contributing/#how-to-become-a-committer",
- "relUrl": "/archives/v1.2.1/docs/configuration/#velox-parameters"
+ "relUrl": "/contributing/#how-to-become-a-committer"
},"9": {
- "doc": "Configuration(v1.2.1)",
- "title": "Configuration(v1.2.1)",
+ "doc": "Contributing to Gluten",
+ "title": "Contributing to Gluten",
"content": " ",
- "url": "/archives/v1.2.1/docs/configuration/",
+ "url": "/contributing/",
- "relUrl": "/archives/v1.2.1/docs/configuration/"
+ "relUrl": "/contributing/"
},"10": {
- "doc": "Configuration",
- "title": "Spark Configurations for Gluten Plugin",
- "content": "There are many configurations could impact the Gluten Plugin performance and can be fine-tuned in Spark. You can add these configurations into spark-defaults.conf to enable or disable the setting. ",
- "url": "/docs/configuration/#spark-configurations-for-gluten-plugin",
+ "doc": "Downloads",
+ "title": "Download Apache Gluten (incubating)",
+ "content": "Gluten is a plugin for Apache Spark to double SparkSQL’s performance. ",
+ "url": "/downloads/#download-apache-gluten-incubating",
- "relUrl": "/docs/configuration/#spark-configurations-for-gluten-plugin"
+ "relUrl": "/downloads/#download-apache-gluten-incubating"
},"11": {
- "doc": "Configuration",
- "title": "Spark parameters",
- "content": "| Parameters | Description | Recommend Setting | . | spark.driver.extraClassPath | To add Gluten Plugin jar file in Spark Driver | /path/to/jar_file | . | spark.executor.extraClassPath | To add Gluten Plugin jar file in Spark Executor | /path/to/jar_file | . | spark.executor.memory | To set up how much memory to be used for Spark Executor. | | . | spark.memory.offHeap.size | To set up how much memory to be used for Java OffHeap. Please notice Gluten Plugin will leverage [...]
- "url": "/docs/configuration/#spark-parameters",
+ "doc": "Downloads",
+ "title": "Instructions",
+ "content": "Please select a version of Gluten to download from the list below; we recommend opting for the latest release. Please ensure the integrity of the release by verifying the corresponding SHA-512 hashes, signatures, and the project release KEYS. Instructions for verifying the SHA-512 hashes and signatures can be found in the provided guidelines. ",
+ "url": "/downloads/#instructions",
- "relUrl": "/docs/configuration/#spark-parameters"
+ "relUrl": "/downloads/#instructions"
},"12": {
- "doc": "Configuration",
- "title": "Velox Parameters",
- "content": "The following configurations are related to Velox settings. | Parameters | Description | Recommend Setting | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The m [...]
- "url": "/docs/configuration/#velox-parameters",
+ "doc": "Downloads",
+ "title": "The Latest Apache Official Release",
+ "content": "Please be aware that the binary file was compiled on CentOS7 using an Intel(r) Xeon(r) Gold 6252 processor with static linking. While it may operate on most operating systems and Intel SKYLAKE or later platforms, performance is not guaranteed. For the best performance and platform support, we recommend compiling from source code yourself. | Version | Date | Source | Binary | Release Notes | . | 1.3.0 | Jan. 24 2025 | source(sha512,signature) | spark32-binary(sha512,signat [...]
+ "url": "/downloads/#the-latest-apache-official-release",
- "relUrl": "/docs/configuration/#velox-parameters"
+ "relUrl": "/downloads/#the-latest-apache-official-release"
},"13": {
- "doc": "Configuration",
- "title": "Configuration",
- "content": " ",
- "url": "/docs/configuration/",
+ "doc": "Downloads",
+ "title": "All Archived Releases",
+ "content": "| Version | Date | Source | Binary | Release Notes | . | 1.1.1(non-Apache release) | Mar. 2 2024 | source | binary location | release notes | . | 1.1.0(non-Apache release) | Nov. 30 2023 | source | binary location | release notes | . | 1.0.0(non-Apache release) | Jul. 13 2023 | source | binary location | release notes | . | 0.5.0(non-Apache release) | Apr. 7 2023 | source | binary location | release notes | . ",
+ "url": "/downloads/#all-archived-releases",
- "relUrl": "/docs/configuration/"
+ "relUrl": "/downloads/#all-archived-releases"
},"14": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Gluten CPP Core Guidelines",
- "content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines",
+ "doc": "Downloads",
+ "title": "License",
+ "content": "The software licensed under Apache License 2.0. ",
+ "url": "/downloads/#license",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines"
+ "relUrl": "/downloads/#license"
},"15": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Philosophy",
- "content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#philosophy",
+ "doc": "Downloads",
+ "title": "Downloads",
+ "content": " ",
+ "url": "/downloads/",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#philosophy"
+ "relUrl": "/downloads/"
},"16": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Code Formatting",
- "content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-formatting",
+ "doc": "Home",
+ "title": "Overview",
+ "content": "Apache Gluten(incubating) is a middle layer responsible for offloading JVM-based SQL engines’ execution to native engines. ",
+ "url": "/#overview",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-formatting"
+ "relUrl": "/#overview"
},"17": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Naming Conventions",
- "content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#naming-conventions",
+ "doc": "Home",
+ "title": "1 Introduction",
+ "content": " ",
+ "url": "/#1-introduction",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#naming-conventions"
+ "relUrl": "/#1-introduction"
},"18": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Designs",
- "content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#designs",
+ "doc": "Home",
+ "title": "1.1 Problem Statement",
+ "content": "Apache Spark is a stable, mature project that has been developed for many years. It is one of the best frameworks to scale out for processing petabyte-scale datasets. However, the Spark community has had to address performance challenges that require various optimizations over time. As a key optimization in Spark 2.0, Whole Stage Code Generation is introduced to replace Volcano Model, which achieves 2x speedup. Henceforth, most optimizations are at query plan level. Singl [...]
+ "url": "/#11-problem-statement",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#designs"
+ "relUrl": "/#11-problem-statement"
},"19": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Source File & Header File",
- "content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#source-file--header-file",
+ "doc": "Home",
+ "title": "1.2 Gluten’s Solution",
+ "content": "“Gluten” is Latin for glue. The main goal of Gluten project is to “glue” native libraries with SparkSQL. Thus, we can benefit from high scalability of Spark SQL framework and high performance of native libraries. The basic rule of Gluten’s design is that we would reuse spark’s whole control flow and as many JVM code as possible but offload the compute-intensive data processing part to native code. Here is what Gluten does: . | Transform Spark’s whole stage physical plan t [...]
+ "url": "/#12-glutens-solution",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#source-file--header-file"
+ "relUrl": "/#12-glutens-solution"
},"20": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Class",
- "content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#class",
+ "doc": "Home",
+ "title": "1.3 Target User",
+ "content": "Gluten’s target user is anyone who wants to accelerate SparkSQL fundamentally. As a plugin to Spark, Gluten doesn’t require any change for dataframe API or SQL query, but only requires user to make correct configuration. See Gluten configuration properties here. ",
+ "url": "/#13-target-user",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#class"
+ "relUrl": "/#13-target-user"
},"21": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Function",
- "content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#function",
+ "doc": "Home",
+ "title": "1.4 References",
+ "content": "You can click below links for more related information. | Gluten References | . ",
+ "url": "/#14-references",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#function"
+ "relUrl": "/#14-references"
},"22": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Variable",
- "content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#variable",
+ "doc": "Home",
+ "title": "2 Architecture",
+ "content": "The overview chart is like below. Substrait provides a well-defined cross-language specification for data compute operations (see more details here). Spark physical plan is transformed to Substrait plan. Then Substrait plan is passed to native through JNI call. On native side, the native operator chain will be built out and offloaded to native engine. Gluten will return Columnar Batch to Spark and Spark Columnar API (since Spark-3.0) will be used at execution time. Gluten [...]
+ "url": "/#2-architecture",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#variable"
+ "relUrl": "/#2-architecture"
},"23": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Constant",
- "content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#constant",
+ "doc": "Home",
+ "title": "3 How to Use",
+ "content": "There are two methods for utilizing Gluten. The first is to use a pre-built JAR for a quick test of Gluten’s acceleration capabilities. The second is to compile the JAR yourself, ensuring it runs on your target platform and delivers the best possible performance. ",
+ "url": "/#3-how-to-use",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#constant"
+ "relUrl": "/#3-how-to-use"
},"24": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Macro",
- "content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#macro",
+ "doc": "Home",
+ "title": "3.1 Use a pre-Built Jar",
+ "content": "One way is to use released jar. Here is a simple example. Currently, only centos7/8 and ubuntu20.04/22.04 are well supported. You can find a pre-built gluten release jar in Apache Gluten Download. Please be aware that the pre-built JAR is compiled using static linking via vcpkg, based on the Intel® Xeon® Gold 6252 processor. It should be compatible with most operating systems, including CentOS and Ubuntu, although performance is not guaranteed. For the optimal performance [...]
+ "url": "/#31-use-a-pre-built-jar",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#macro"
+ "relUrl": "/#31-use-a-pre-built-jar"
},"25": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Namespace",
- "content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#namespace",
+ "doc": "Home",
+ "title": "3.2 Custom Build",
+ "content": "To utilize Gluten with Spark, you can compile Gluten from the source and then configure it to enable the Gluten plugin. Below is a straightforward example. For more comprehensive instructions, please refer to the detailed guidance provided in the corresponding backend section. export gluten_jar = /PATH/TO/GLUTEN/backends-velox/target/<gluten-jar> spark-shell \\ --master yarn --deploy-mode client \\ --conf spark.plugins=org.apache.gluten.GlutenPlugin \\ --conf spark. [...]
+ "url": "/#32-custom-build",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#namespace"
+ "relUrl": "/#32-custom-build"
},"26": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Resource Management",
- "content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#resource-management",
+ "doc": "Home",
+ "title": "3.2.1 Build and install Gluten with Velox backend",
+ "content": "If you want to use Gluten Velox backend, see Build with Velox to build and install the necessary libraries. You can also find more information in Gluten with Velox backend and velox. ",
+ "url": "/#321-build-and-install-gluten-with-velox-backend",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#resource-management"
+ "relUrl": "/#321-build-and-install-gluten-with-velox-backend"
},"27": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Exception",
- "content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#exception",
+ "doc": "Home",
+ "title": "3.2.2 Build and install Gluten with ClickHouse backend",
+ "content": "If you want to use Gluten ClickHouse backend, see Build with ClickHouse Backend. ClickHouse backend is developed by Kyligence, please visit Kyligence’s ClickHouse for more infomation. ",
+ "url": "/#322-build-and-install-gluten-with-clickhouse-backend",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#exception"
+ "relUrl": "/#322-build-and-install-gluten-with-clickhouse-backend"
},"28": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "Code Comment",
- "content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-comment",
+ "doc": "Home",
+ "title": "3.2.3 Build options",
+ "content": "See Build Parameters. ",
+ "url": "/#323-build-options",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-comment"
+ "relUrl": "/#323-build-options"
},"29": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "References",
- "content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#references",
+ "doc": "Home",
+ "title": "4 Join the Community",
+ "content": " ",
+ "url": "/#4-join-the-community",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#references"
+ "relUrl": "/#4-join-the-community"
},"30": {
- "doc": "CPP Code Style(v1.2.1)",
- "title": "CPP Code Style(v1.2.1)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/developers/cpp-code-style/",
+ "doc": "Home",
+ "title": "Contact Us",
+ "content": "Gluten was initiated by Intel and Kyligence in 2022. Several companies are also actively participating in the development, such as BIGO, Meituan, Alibaba Cloud, NetEase, Baidu, Microsoft, etc. If you are interested in Gluten project, please contact and subscribe below mailing lists for further discussion. Please see contact us for more information. ",
+ "url": "/#contact-us",
- "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/"
+ "relUrl": "/#contact-us"
},"31": {
+ "doc": "Home",
+ "title": "Source Code",
+ "content": "Please see gluten source code for more information. ",
+ "url": "/#source-code",
+
+ "relUrl": "/#source-code"
+ },"32": {
+ "doc": "Home",
+ "title": "How to Contribute to Gluten",
+ "content": "Please see contributing guide about how to make contributions. ",
+ "url": "/#how-to-contribute-to-gluten",
+
+ "relUrl": "/#how-to-contribute-to-gluten"
+ },"33": {
+ "doc": "Home",
+ "title": "Wechat Group",
+ "content": "There is a Wechat group (in Chinese) which may be more friendly for PRC developers/users. Due to the limitation of wechat group, please mail to dev@gluten.apache.org. ",
+ "url": "/#wechat-group",
+
+ "relUrl": "/#wechat-group"
+ },"34": {
+ "doc": "Home",
+ "title": "Slack channel",
+ "content": "For Velox backend, we recommend to use velox community. ",
+ "url": "/#slack-channel",
+
+ "relUrl": "/#slack-channel"
+ },"35": {
+ "doc": "Home",
+ "title": "5 Performance",
+ "content": "We use TPCH-like and TPCDS-like as decison support benchmarks to evaluate Gluten’s performance. TPCH-like is a query set for modified from TPC-H benchmark and TPCDS-like is a query set for modified from TPC-DS benchmark. We use Parquet file format for Velox testing & MergeTree file format for Clickhouse testing, compared to Parquet file format as baseline. See Decision Support Benchmark. ",
+ "url": "/#5-performance",
+
+ "relUrl": "/#5-performance"
+ },"36": {
+ "doc": "Home",
+ "title": "Gluten + Velox backend Performance",
+ "content": "The below test environment: single node with 3TB data on Intel® Xeon® Platinum 8592+; Spark-3.3.1 for both baseline and Gluten. The TPCH-like result (tested in Mar. 2024) shows an overall speedup of 3.34x and up to 23.45x speedup in a single query with Gluten + Velox backend used. The TPCDS-like result (tested in Mar. 2024) shows an overall speedup of 3.02x and up to 13.75x speedup in a single query with Gluten + Velox backend used. Notices & Disclaimers . Performance [...]
+ "url": "/#gluten--velox-backend-performance",
+
+ "relUrl": "/#gluten--velox-backend-performance"
+ },"37": {
+ "doc": "Home",
+ "title": "Gluten + ClickHouse backend Performance",
+ "content": "The below testing environment: a 8-nodes AWS cluster with 1TB data; Spark-3.1.1 for both baseline and Gluten. The Decision Support Benchmark1 result shows an average speedup of 2.12x and up to 3.48x speedup with Gluten Clickhouse backend. ",
+ "url": "/#gluten--clickhouse-backend-performance",
+
+ "relUrl": "/#gluten--clickhouse-backend-performance"
+ },"38": {
+ "doc": "Home",
+ "title": "Thanks to our contributors",
+ "content": ". ",
+ "url": "/#thanks-to-our-contributors",
+
+ "relUrl": "/#thanks-to-our-contributors"
+ },"39": {
+ "doc": "Home",
+ "title": "Home",
+ "content": " ",
+ "url": "/",
+
+ "relUrl": "/"
+ },"40": {
+ "doc": "Powered by Gluten",
+ "title": "Project and Product Name using “Gluten”",
+ "content": "Organizations developing products and projects for use with Apache Gluten (incubating), as well as related marketing materials, should ensure they respect the trademark of “Apache Gluten” and its logo. For comprehensive and authoritative guidance on the proper usage of ASF trademarks, please refer to the ASF Trademarks Guidance and the associated FAQ. ",
+ "url": "/poweredby/#project-and-product-name-using-gluten",
+
+ "relUrl": "/poweredby/#project-and-product-name-using-gluten"
+ },"41": {
+ "doc": "Powered by Gluten",
+ "title": "Companies and Organizations",
+ "content": "To add yourself to the list, please email dev@gluten.apache.org with the following information: your organization name, URL, a list of the Gluten components you are using, and a brief description of your use case. Sorted by Company Name . | Company Name | Description | Link | . | Alibaba | Alibaba has chosen Gluten as a part of their public cloud solution. | | . | Baidu | Baidu has used gluten in multiple business scenarios. | | . | Bytedance | EMR in Bytedance’s Volc [...]
+ "url": "/poweredby/#companies-and-organizations",
+
+ "relUrl": "/poweredby/#companies-and-organizations"
+ },"42": {
+ "doc": "Powered by Gluten",
+ "title": "Powered by Gluten",
+ "content": " ",
+ "url": "/poweredby/",
+
+ "relUrl": "/poweredby/"
+ },"43": {
+ "doc": "Gluten References",
+ "title": "Gluten related Use Cases and Publications",
+ "content": "For more information on Apache Gluten (incubating), including related articles and videos, please refer to the following links. | Title | Company | Publish Date | Reference | . | Gluten: A middle layer to offload Spark SQL to Native | Intel | Jul. 2022 | Link | . | Best Exploration of Columnar Shuffle Design | Intel | Jul. 2022 | Link | . | Gluten Introduction | Intel | Sept. 2022 | Link | . | Introducing Velox | Meta | Mar. 2023 | Link | . | Gluten in Kyligence(in Chines [...]
+ "url": "/references/#gluten-related-use-cases-and-publications",
+
+ "relUrl": "/references/#gluten-related-use-cases-and-publications"
+ },"44": {
+ "doc": "Gluten References",
+ "title": "Gluten References",
+ "content": " ",
+ "url": "/references/",
+
+ "relUrl": "/references/"
+ },"45": {
+ "doc": "Archives",
+ "title": "Gluten Documents by version",
+ "content": " ",
+ "url": "/archives/#gluten-documents-by-version",
+
+ "relUrl": "/archives/#gluten-documents-by-version"
+ },"46": {
+ "doc": "Archives",
+ "title": "Archives",
+ "content": " ",
+ "url": "/archives/",
+
+ "relUrl": "/archives/"
+ },"47": {
+ "doc": "Documentation(Latest)",
+ "title": "Documentation(Latest)",
+ "content": " Click here to view the latest documentation . If you are not redirected, click here. ",
+ "url": "/docs/latest.html",
+
+ "relUrl": "/docs/latest.html"
+ },"48": {
"doc": "v1.1.1/CPP Code Style",
"title": "Gluten CPP Core Guidelines",
"content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#gluten-cpp-core-guidelines",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#gluten-cpp-core-guidelines"
- },"32": {
+ },"49": {
"doc": "v1.1.1/CPP Code Style",
"title": "Philosophy",
"content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#philosophy",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#philosophy"
- },"33": {
+ },"50": {
"doc": "v1.1.1/CPP Code Style",
"title": "Code Formatting",
"content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#code-formatting",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#code-formatting"
- },"34": {
+ },"51": {
"doc": "v1.1.1/CPP Code Style",
"title": "Naming Conventions",
"content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#naming-conventions",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#naming-conventions"
- },"35": {
+ },"52": {
"doc": "v1.1.1/CPP Code Style",
"title": "Designs",
"content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#designs",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#designs"
- },"36": {
+ },"53": {
"doc": "v1.1.1/CPP Code Style",
"title": "Source File & Header File",
"content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#source-file--header-file",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#source-file--header-file"
- },"37": {
+ },"54": {
"doc": "v1.1.1/CPP Code Style",
"title": "Class",
"content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#class",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#class"
- },"38": {
+ },"55": {
"doc": "v1.1.1/CPP Code Style",
"title": "Function",
"content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#function",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#function"
- },"39": {
+ },"56": {
"doc": "v1.1.1/CPP Code Style",
"title": "Variable",
"content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#variable",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#variable"
- },"40": {
+ },"57": {
"doc": "v1.1.1/CPP Code Style",
"title": "Constant",
"content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#constant",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#constant"
- },"41": {
+ },"58": {
"doc": "v1.1.1/CPP Code Style",
"title": "Macro",
"content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#macro",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#macro"
- },"42": {
+ },"59": {
"doc": "v1.1.1/CPP Code Style",
"title": "Namespace",
"content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#namespace",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#namespace"
- },"43": {
+ },"60": {
"doc": "v1.1.1/CPP Code Style",
"title": "Resource Management",
"content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#resource-management",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#resource-management"
- },"44": {
+ },"61": {
"doc": "v1.1.1/CPP Code Style",
"title": "Exception",
"content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#exception",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#exception"
- },"45": {
+ },"62": {
"doc": "v1.1.1/CPP Code Style",
"title": "Code Comment",
"content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#code-comment",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#code-comment"
- },"46": {
+ },"63": {
"doc": "v1.1.1/CPP Code Style",
"title": "References",
"content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#references",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#references"
- },"47": {
+ },"64": {
"doc": "v1.1.1/CPP Code Style",
"title": "v1.1.1/CPP Code Style",
"content": " ",
"url": "/archives/v1.1.1/developers/cpp_code_style/",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/"
- },"48": {
- "doc": "CPP Code Style",
- "title": "Gluten CPP Core Guidelines",
- "content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
- "url": "/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines",
+ },"65": {
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "How to understand the key work of Gluten?",
+ "content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
+ "url": "/archives/v1.1.1/developers/howto/#how-to-understand-the-key-work-of-gluten",
- "relUrl": "/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines"
- },"49": {
- "doc": "CPP Code Style",
- "title": "Philosophy",
- "content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
- "url": "/docs/developers/cpp-code-style/#philosophy",
+ "relUrl": "/archives/v1.1.1/developers/howto/#how-to-understand-the-key-work-of-gluten"
+ },"66": {
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "How to debug in Gluten?",
+ "content": " ",
+ "url": "/archives/v1.1.1/developers/howto/#how-to-debug-in-gluten",
- "relUrl": "/docs/developers/cpp-code-style/#philosophy"
- },"50": {
- "doc": "CPP Code Style",
- "title": "Code Formatting",
- "content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
- "url": "/docs/developers/cpp-code-style/#code-formatting",
+ "relUrl": "/archives/v1.1.1/developers/howto/#how-to-debug-in-gluten"
+ },"67": {
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "1 How to debug C++",
+ "content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
+ "url": "/archives/v1.1.1/developers/howto/#1-how-to-debug-c",
- "relUrl": "/docs/developers/cpp-code-style/#code-formatting"
- },"51": {
- "doc": "CPP Code Style",
- "title": "Naming Conventions",
- "content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
- "url": "/docs/developers/cpp-code-style/#naming-conventions",
+ "relUrl": "/archives/v1.1.1/developers/howto/#1-how-to-debug-c"
+ },"68": {
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "2 How to debug plan validation process",
+ "content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
+ "url": "/archives/v1.1.1/developers/howto/#2-how-to-debug-plan-validation-process",
- "relUrl": "/docs/developers/cpp-code-style/#naming-conventions"
- },"52": {
- "doc": "CPP Code Style",
- "title": "Designs",
- "content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
- "url": "/docs/developers/cpp-code-style/#designs",
+ "relUrl": "/archives/v1.1.1/developers/howto/#2-how-to-debug-plan-validation-process"
+ },"69": {
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "3 How to debug Java/Scala",
+ "content": "wait to add . ",
+ "url": "/archives/v1.1.1/developers/howto/#3-how-to-debug-javascala",
- "relUrl": "/docs/developers/cpp-code-style/#designs"
- },"53": {
- "doc": "CPP Code Style",
- "title": "Source File & Header File",
- "content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
- "url": "/docs/developers/cpp-code-style/#source-file--header-file",
+ "relUrl": "/archives/v1.1.1/developers/howto/#3-how-to-debug-javascala"
+ },"70": {
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "4 How to debug with core-dump",
+ "content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
+ "url": "/archives/v1.1.1/developers/howto/#4-how-to-debug-with-core-dump",
- "relUrl": "/docs/developers/cpp-code-style/#source-file--header-file"
- },"54": {
- "doc": "CPP Code Style",
- "title": "Class",
- "content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
- "url": "/docs/developers/cpp-code-style/#class",
+ "relUrl": "/archives/v1.1.1/developers/howto/#4-how-to-debug-with-core-dump"
+ },"71": {
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "How to run TPC-H on Velox backend",
+ "content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox-be by workflow, you can refer to velox_be.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, prepare the d [...]
+ "url": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-h-on-velox-backend",
- "relUrl": "/docs/developers/cpp-code-style/#class"
- },"55": {
- "doc": "CPP Code Style",
- "title": "Function",
- "content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
- "url": "/docs/developers/cpp-code-style/#function",
-
- "relUrl": "/docs/developers/cpp-code-style/#function"
- },"56": {
- "doc": "CPP Code Style",
- "title": "Variable",
- "content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
- "url": "/docs/developers/cpp-code-style/#variable",
-
- "relUrl": "/docs/developers/cpp-code-style/#variable"
- },"57": {
- "doc": "CPP Code Style",
- "title": "Constant",
- "content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
- "url": "/docs/developers/cpp-code-style/#constant",
-
- "relUrl": "/docs/developers/cpp-code-style/#constant"
- },"58": {
- "doc": "CPP Code Style",
- "title": "Macro",
- "content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
- "url": "/docs/developers/cpp-code-style/#macro",
-
- "relUrl": "/docs/developers/cpp-code-style/#macro"
- },"59": {
- "doc": "CPP Code Style",
- "title": "Namespace",
- "content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
- "url": "/docs/developers/cpp-code-style/#namespace",
-
- "relUrl": "/docs/developers/cpp-code-style/#namespace"
- },"60": {
- "doc": "CPP Code Style",
- "title": "Resource Management",
- "content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
- "url": "/docs/developers/cpp-code-style/#resource-management",
-
- "relUrl": "/docs/developers/cpp-code-style/#resource-management"
- },"61": {
- "doc": "CPP Code Style",
- "title": "Exception",
- "content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
- "url": "/docs/developers/cpp-code-style/#exception",
-
- "relUrl": "/docs/developers/cpp-code-style/#exception"
- },"62": {
- "doc": "CPP Code Style",
- "title": "Code Comment",
- "content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
- "url": "/docs/developers/cpp-code-style/#code-comment",
-
- "relUrl": "/docs/developers/cpp-code-style/#code-comment"
- },"63": {
- "doc": "CPP Code Style",
- "title": "References",
- "content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
- "url": "/docs/developers/cpp-code-style/#references",
-
- "relUrl": "/docs/developers/cpp-code-style/#references"
- },"64": {
- "doc": "CPP Code Style",
- "title": "CPP Code Style",
- "content": " ",
- "url": "/docs/developers/cpp-code-style/",
-
- "relUrl": "/docs/developers/cpp-code-style/"
- },"65": {
- "doc": "v1.1.1/Getting Started with ClickHouse Backend",
- "title": "ClickHouse Backend",
- "content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
- "url": "/archives/v1.1.1/docs/clickhouse/getting-started/#clickhouse-backend",
-
- "relUrl": "/archives/v1.1.1/docs/clickhouse/getting-started/#clickhouse-backend"
- },"66": {
- "doc": "v1.1.1/Getting Started with ClickHouse Backend",
- "title": "v1.1.1/Getting Started with ClickHouse Backend",
- "content": " ",
- "url": "/archives/v1.1.1/docs/clickhouse/getting-started/",
-
- "relUrl": "/archives/v1.1.1/docs/clickhouse/getting-started/"
- },"67": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Supported Version",
- "content": "| Type | Version |——-|——————————| Spark | 3.2.2, 3.3.1 | OS | Ubuntu20.04/22.04, Centos7/8 | jdk | openjdk8 | scala | 2.12 . Spark3.4.0 support is still WIP. TPCH/DS can pass, UT is not yet passed. There are pending PRs for jdk11 support. Currently, the mvn script can automatically fetch and build all dependency libraries incluing Velox. Our nightly build still use Velox under oap-project. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#supported-version",
-
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#supported-version"
- },"68": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Prerequisite",
- "content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Gluten builds with Spark3.2.x and Spark3.3.x now but only fully tested in CI with 3.2.2 and 3.3.1. We will add/update supported/tested versions according to the upstream changes. we need to set up the JAVA_HOME env. Currently, java 8 is required and the support for j [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#prerequisite",
-
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#prerequisite"
- },"69": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Build Gluten with Velox Backend",
- "content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . cd /path/to/gluten ## The script builds two jars for spark 3.2.2 and 3.3.1./dev/buildbundle-veloxbe.sh ## After a complete build, if you need to re-build the project and only some gluten code is changed, ## you can use the following command to skip building velox and protobuf. [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-velox-backend",
-
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-velox-backend"
- },"70": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Dependency library deployment",
- "content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#dependency-library-deployment",
-
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#dependency-library-deployment"
- },"71": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "HDFS support",
- "content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#hdfs-support",
-
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#hdfs-support"
+ "relUrl": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-h-on-velox-backend"
},"72": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "AWS S3 support",
- "content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#aws-s3-support",
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "How to run TPC-DS",
+ "content": "wait to add . ",
+ "url": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-ds",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#aws-s3-support"
+ "relUrl": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-ds"
},"73": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Celeborn support",
- "content": "Gluten with velox backend supports Celeborn as remote shuffle service. Below introduction is used to enable this feature . First refer to this URL(https://github.com/apache/incubator-celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable rss profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Prss -DskipTests . Then add the Gluten and Spark Celeborn Client packages to your Spark application’s classpath(usua [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#celeborn-support",
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "How to track the memory exhaust problem",
+ "content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
+ "url": "/archives/v1.1.1/developers/howto/#how-to-track-the-memory-exhaust-problem",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#celeborn-support"
+ "relUrl": "/archives/v1.1.1/developers/howto/#how-to-track-the-memory-exhaust-problem"
},"74": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "DeltaLake Support",
- "content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with velox backends [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#deltalake-support",
+ "doc": "v1.1.1/How To Use Gluten",
+ "title": "v1.1.1/How To Use Gluten",
+ "content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
+ "url": "/archives/v1.1.1/developers/howto/",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#deltalake-support"
+ "relUrl": "/archives/v1.1.1/developers/howto/"
},"75": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Iceberg Support",
- "content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Then, put the additional gluten-iceberg jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-iceberg jar is in gluten-iceberg/target directory. After the two steps, you can query icebe [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#iceberg-support",
+ "doc": "v1.1.1/How To Release",
+ "title": "How to Release",
+ "content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
+ "url": "/archives/v1.1.1/developers/howtorelease/#how-to-release",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#iceberg-support"
+ "relUrl": "/archives/v1.1.1/developers/howtorelease/#how-to-release"
},"76": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Coverage",
- "content": "Spark3.3 has 387 functions in total. ~240 are commonly used. Velox’s functions have two category, Presto and Spark. Presto has 124 functions implemented. Spark has 62 functions. Spark functions are verified to have the same result as Vanilla Spark. Some Presto functions have the same result as Vanilla Spark but some others have different. Gluten prefer to use Spark functions firstly. If it’s not in Spark’s list but implemented in Presto, we currently offload to Presto one [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#coverage",
+ "doc": "v1.1.1/How To Release",
+ "title": "Release Process",
+ "content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
+ "url": "/archives/v1.1.1/developers/howtorelease/#release-process",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#coverage"
+ "relUrl": "/archives/v1.1.1/developers/howtorelease/#release-process"
},"77": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Spill (Experimental)",
- "content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#spill-experimental",
+ "doc": "v1.1.1/How To Release",
+ "title": "v1.1.1/How To Release",
+ "content": " ",
+ "url": "/archives/v1.1.1/developers/howtorelease/",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#spill-experimental"
+ "relUrl": "/archives/v1.1.1/developers/howtorelease/"
},"78": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Velox User-Defined Functions (UDF)",
- "content": " ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#velox-user-defined-functions-udf",
+ "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
+ "title": "Generate Micro Benchmarks for Velox Backend",
+ "content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
+ "url": "/archives/v1.1.1/developers/microbenchmark/#generate-micro-benchmarks-for-velox-backend",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#velox-user-defined-functions-udf"
+ "relUrl": "/archives/v1.1.1/developers/microbenchmark/#generate-micro-benchmarks-for-velox-backend"
},"79": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Introduction",
- "content": "Velox backend supports User-Defined Functions (UDF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#introduction",
+ "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
+ "title": "Try the example",
+ "content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
+ "url": "/archives/v1.1.1/developers/microbenchmark/#try-the-example",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#introduction"
+ "relUrl": "/archives/v1.1.1/developers/microbenchmark/#try-the-example"
},"80": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Creating a UDF library",
- "content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#creating-a-udf-library",
+ "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
+ "title": "Generate Substrait plan and input for any query",
+ "content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
+ "url": "/archives/v1.1.1/developers/microbenchmark/#generate-substrait-plan-and-input-for-any-query",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#creating-a-udf-library"
+ "relUrl": "/archives/v1.1.1/developers/microbenchmark/#generate-substrait-plan-and-input-for-any-query"
},"81": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Building the UDF library",
- "content": "To build the UDF library, users need to compile the C++ code and link to libvelox.so. It’s recommended to create a CMakeLists.txt for the project. Here’s an example: . project(myudf) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(GLUTEN_HOME /path/to/gluten) add_library(myudf SHARED \"MyUDF.cpp\") find_library(VELOX_LIBRARY REQUIRED NAMES velox HINTS ${GLUTEN_HOME}/cpp/build/releases NO_DEFAULT_PATH) target_include_directories(myudf PRIVATE ${GLUTEN_HO [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#building-the-udf-library",
+ "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
+ "title": "Save ouput to parquet to analyze",
+ "content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
+ "url": "/archives/v1.1.1/developers/microbenchmark/#save-ouput-to-parquet-to-analyze",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#building-the-udf-library"
+ "relUrl": "/archives/v1.1.1/developers/microbenchmark/#save-ouput-to-parquet-to-analyze"
},"82": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Using UDF in Gluten",
- "content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the libray paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#using-udf-in-gluten",
+ "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
+ "title": "Add shuffle write process",
+ "content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
+ "url": "/archives/v1.1.1/developers/microbenchmark/#add-shuffle-write-process",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#using-udf-in-gluten"
+ "relUrl": "/archives/v1.1.1/developers/microbenchmark/#add-shuffle-write-process"
},"83": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Try the example",
- "content": "We provided an Velox UDF example file MyUDF.cpp. After building gluten cpp, you can find the example library at /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so . Start spark-shell or spark-sql with below configuration . --files /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so --conf spark.gluten.sql.columnar.backend.velox.udfLibraryPaths=libmyudf.so . Run query. The functions myudf1 and myudf2 increment the input value by a constant of 5 . select myudf1(1 [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#try-the-example",
+ "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
+ "title": "Simulate Spark with multiple processes and threads",
+ "content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
+ "url": "/archives/v1.1.1/developers/microbenchmark/#simulate-spark-with-multiple-processes-and-threads",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#try-the-example"
+ "relUrl": "/archives/v1.1.1/developers/microbenchmark/#simulate-spark-with-multiple-processes-and-threads"
},"84": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "High-Bandwidth Memory (HBM) support",
- "content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#high-bandwidth-memory-hbm-support",
+ "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
+ "title": "v1.1.1/Micro Benchmarks for Velox Backend",
+ "content": " ",
+ "url": "/archives/v1.1.1/developers/microbenchmark/",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#high-bandwidth-memory-hbm-support"
+ "relUrl": "/archives/v1.1.1/developers/microbenchmark/"
},"85": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Build Gluten with HBM",
- "content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds two jars for spark 3.2.2 and 3.3.1./dev/buildbundle-veloxbe.sh --enable_hbm=ON . ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-hbm",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Environment",
+ "content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
+ "url": "/archives/v1.1.1/developers/newto/#environment",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-hbm"
+ "relUrl": "/archives/v1.1.1/developers/newto/#environment"
},"86": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Configure and enable HBM in Spark Application",
- "content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#configure-and-enable-hbm-in-spark-application",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Openjdk8",
+ "content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
+ "url": "/archives/v1.1.1/developers/newto/#openjdk8",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#configure-and-enable-hbm-in-spark-application"
+ "relUrl": "/archives/v1.1.1/developers/newto/#openjdk8"
},"87": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Intel® QuickAssist Technology (QAT) support",
- "content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#intel-quickassist-technology-qat-support",
-
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#intel-quickassist-technology-qat-support"
- },"88": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Software Requirements",
- "content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#software-requirements",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Maven 3.6.3 or above",
+ "content": "Maven Dowload Page And then set the environment setting. ",
+ "url": "/archives/v1.1.1/developers/newto/#maven-363-or-above",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#software-requirements"
+ "relUrl": "/archives/v1.1.1/developers/newto/#maven-363-or-above"
+ },"88": {
+ "doc": "v1.1.1/New To Gluten",
+ "title": "GCC 9.4 or above",
+ "content": " ",
+ "url": "/archives/v1.1.1/developers/newto/#gcc-94-or-above",
+
+ "relUrl": "/archives/v1.1.1/developers/newto/#gcc-94-or-above"
},"89": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Build Gluten with QAT",
- "content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-qat",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Compile gluten using debug mode",
+ "content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
+ "url": "/archives/v1.1.1/developers/newto/#compile-gluten-using-debug-mode",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-qat"
+ "relUrl": "/archives/v1.1.1/developers/newto/#compile-gluten-using-debug-mode"
},"90": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Enable QAT with Gzip/Zstd for shuffle compression",
- "content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#enable-qat-with-gzipzstd-for-shuffle-compression",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Java/scala code development with Intellij",
+ "content": " ",
+ "url": "/archives/v1.1.1/developers/newto/#javascala-code-development-with-intellij",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#enable-qat-with-gzipzstd-for-shuffle-compression"
+ "relUrl": "/archives/v1.1.1/developers/newto/#javascala-code-development-with-intellij"
},"91": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "QAT driver references",
- "content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#qat-driver-references",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Linux intellij local debug",
+ "content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
+ "url": "/archives/v1.1.1/developers/newto/#linux-intellij-local-debug",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#qat-driver-references"
+ "relUrl": "/archives/v1.1.1/developers/newto/#linux-intellij-local-debug"
},"92": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
- "content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#intel-in-memory-analytics-accelerator-iaaiax-support",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Windows/Mac intellij remote debug",
+ "content": "If you have Ultimate intellij, you can try to debug remotely. ",
+ "url": "/archives/v1.1.1/developers/newto/#windowsmac-intellij-remote-debug",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#intel-in-memory-analytics-accelerator-iaaiax-support"
+ "relUrl": "/archives/v1.1.1/developers/newto/#windowsmac-intellij-remote-debug"
},"93": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Build Gluten with IAA",
- "content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-iaa",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Set up gluten project",
+ "content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
+ "url": "/archives/v1.1.1/developers/newto/#set-up-gluten-project",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-iaa"
+ "relUrl": "/archives/v1.1.1/developers/newto/#set-up-gluten-project"
},"94": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Enable IAA with Gzip Compression for shuffle compression",
- "content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#enable-iaa-with-gzip-compression-for-shuffle-compression",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Java/Scala code style",
+ "content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
+ "url": "/archives/v1.1.1/developers/newto/#javascala-code-style",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#enable-iaa-with-gzip-compression-for-shuffle-compression"
+ "relUrl": "/archives/v1.1.1/developers/newto/#javascala-code-style"
},"95": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "IAA references",
- "content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#iaa-references",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "CPP code development with Visual Studio Code",
+ "content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
+ "url": "/archives/v1.1.1/developers/newto/#cpp-code-development-with-visual-studio-code",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#iaa-references"
+ "relUrl": "/archives/v1.1.1/developers/newto/#cpp-code-development-with-visual-studio-code"
},"96": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
- "content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Usage",
+ "content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
+ "url": "/archives/v1.1.1/developers/newto/#usage",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
+ "relUrl": "/archives/v1.1.1/developers/newto/#usage"
},"97": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Data preparation",
- "content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#data-preparation",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Usefule notes",
+ "content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
+ "url": "/archives/v1.1.1/developers/newto/#usefule-notes",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#data-preparation"
+ "relUrl": "/archives/v1.1.1/developers/newto/#usefule-notes"
},"98": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Submit the Spark SQL job",
- "content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#submit-the-spark-sql-job",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Debug cpp code with coredump",
+ "content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
+ "url": "/archives/v1.1.1/developers/newto/#debug-cpp-code-with-coredump",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#submit-the-spark-sql-job"
+ "relUrl": "/archives/v1.1.1/developers/newto/#debug-cpp-code-with-coredump"
},"99": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Result",
- "content": "wholestagetransformer indicates that the offload works. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#result",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Debug cpp with gdb",
+ "content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
+ "url": "/archives/v1.1.1/developers/newto/#debug-cpp-with-gdb",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#result"
+ "relUrl": "/archives/v1.1.1/developers/newto/#debug-cpp-with-gdb"
},"100": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Performance",
- "content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#performance",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Run TPC-H and TPC-DS",
+ "content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_be.yml . ",
+ "url": "/archives/v1.1.1/developers/newto/#run-tpc-h-and-tpc-ds",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#performance"
+ "relUrl": "/archives/v1.1.1/developers/newto/#run-tpc-h-and-tpc-ds"
},"101": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "External reference setup",
- "content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#external-reference-setup",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "Run gluten+velox on clean machine",
+ "content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
+ "url": "/archives/v1.1.1/developers/newto/#run-glutenvelox-on-clean-machine",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#external-reference-setup"
+ "relUrl": "/archives/v1.1.1/developers/newto/#run-glutenvelox-on-clean-machine"
},"102": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Gluten UI",
- "content": " ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-ui",
+ "doc": "v1.1.1/New To Gluten",
+ "title": "v1.1.1/New To Gluten",
+ "content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
+ "url": "/archives/v1.1.1/developers/newto/",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-ui"
+ "relUrl": "/archives/v1.1.1/developers/newto/"
},"103": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Gluten event",
- "content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-event",
+ "doc": "v1.1.1/Substrait Modifications",
+ "title": "Substrait Modifications in Gluten",
+ "content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
+ "url": "/archives/v1.1.1/developers/substrait/#substrait-modifications-in-gluten",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-event"
+ "relUrl": "/archives/v1.1.1/developers/substrait/#substrait-modifications-in-gluten"
},"104": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "SQL tab",
- "content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#sql-tab",
+ "doc": "v1.1.1/Substrait Modifications",
+ "title": "Modifications to algebra.proto",
+ "content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
+ "url": "/archives/v1.1.1/developers/substrait/#modifications-to-algebraproto",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#sql-tab"
+ "relUrl": "/archives/v1.1.1/developers/substrait/#modifications-to-algebraproto"
},"105": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "History server",
- "content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#history-server",
+ "doc": "v1.1.1/Substrait Modifications",
+ "title": "Modifications to type.proto",
+ "content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
+ "url": "/archives/v1.1.1/developers/substrait/#modifications-to-typeproto",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#history-server"
+ "relUrl": "/archives/v1.1.1/developers/substrait/#modifications-to-typeproto"
},"106": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Native plan string",
- "content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-string",
+ "doc": "v1.1.1/Substrait Modifications",
+ "title": "v1.1.1/Substrait Modifications",
+ "content": " ",
+ "url": "/archives/v1.1.1/developers/substrait/",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-string"
+ "relUrl": "/archives/v1.1.1/developers/substrait/"
},"107": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Native plan with stats",
- "content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
- "url": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-with-stats",
+ "doc": "v1.1.1/Docker script for CentOS 7",
+ "title": "v1.1.1/Docker script for CentOS 7",
+ "content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ sudo # gluten need maven version >=3.6.3 wget https://downloads.apache.org/m [...]
+ "url": "/archives/v1.1.1/developers/docker_centos7/",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-with-stats"
+ "relUrl": "/archives/v1.1.1/developers/docker_centos7/"
},"108": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "Gluten Implicits",
- "content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-implicits",
+ "doc": "v1.1.1/Docker script for CentOS 8",
+ "title": "v1.1.1/Docker script for CentOS 8",
+ "content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
+ "url": "/archives/v1.1.1/developers/docker_centos8",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-implicits"
+ "relUrl": "/archives/v1.1.1/developers/docker_centos8"
},"109": {
- "doc": "v1.1.1/Getting-Started with Velox Backend",
- "title": "v1.1.1/Getting-Started with Velox Backend",
- "content": " ",
- "url": "/archives/v1.1.1/docs/velox/getting-started/",
+ "doc": "v1.1.1/Docker script for Ubuntu 22.04/20.04",
+ "title": "v1.1.1/Docker script for Ubuntu 22.04/20.04",
+ "content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
+ "url": "/archives/v1.1.1/developers/docker_ubuntu/",
- "relUrl": "/archives/v1.1.1/docs/velox/getting-started/"
+ "relUrl": "/archives/v1.1.1/developers/docker_ubuntu/"
},"110": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "How to understand the key work of Gluten?",
- "content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
- "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten",
+ "doc": "v1.1.1/Developers",
+ "title": "Gluten Developers",
+ "content": "This document provides a developer overview of the project and covers the following topics: . ",
+ "url": "/archives/v1.1.1/developers/#gluten-developers",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten"
+ "relUrl": "/archives/v1.1.1/developers/#gluten-developers"
},"111": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "How to debug in Gluten?",
+ "doc": "v1.1.1/Developers",
+ "title": "v1.1.1/Developers",
"content": " ",
- "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-debug-in-gluten",
+ "url": "/archives/v1.1.1/developers/",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-debug-in-gluten"
+ "relUrl": "/archives/v1.1.1/developers/"
},"112": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "1 How to debug C++",
- "content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
- "url": "/archives/v1.2.1/docs/developers/how-to-use/#1-how-to-debug-c",
+ "doc": "v1.1.1/Velox Function Development",
+ "title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
+ "content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
+ "url": "/archives/v1.1.1/developers/velox_function_development/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#1-how-to-debug-c"
+ "relUrl": "/archives/v1.1.1/developers/velox_function_development/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
},"113": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "2 How to debug plan validation process",
- "content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
- "url": "/archives/v1.2.1/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process",
+ "doc": "v1.1.1/Velox Function Development",
+ "title": "v1.1.1/Velox Function Development",
+ "content": " ",
+ "url": "/archives/v1.1.1/developers/velox_function_development/",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process"
+ "relUrl": "/archives/v1.1.1/developers/velox_function_development/"
},"114": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "3 How to debug Java/Scala",
- "content": "wait to add . ",
- "url": "/archives/v1.2.1/docs/developers/how-to-use/#3-how-to-debug-javascala",
+ "doc": "v1.1.1/ClickHouse Backend",
+ "title": "Gluten Documents by version",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/clickhouse/#gluten-documents-by-version",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#3-how-to-debug-javascala"
+ "relUrl": "/archives/v1.1.1/docs/clickhouse/#gluten-documents-by-version"
},"115": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "4 How to debug with core-dump",
- "content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
- "url": "/archives/v1.2.1/docs/developers/how-to-use/#4-how-to-debug-with-core-dump",
+ "doc": "v1.1.1/ClickHouse Backend",
+ "title": "v1.1.1/ClickHouse Backend",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/clickhouse/",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#4-how-to-debug-with-core-dump"
+ "relUrl": "/archives/v1.1.1/docs/clickhouse/"
},"116": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "How to run TPC-H on Velox backend",
- "content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox with docker by workflow, you can refer to velox_docker.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, [...]
- "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend",
+ "doc": "v1.1.1/Getting Started with ClickHouse Backend",
+ "title": "ClickHouse Backend",
+ "content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
+ "url": "/archives/v1.1.1/docs/clickhouse/getting-started/#clickhouse-backend",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend"
+ "relUrl": "/archives/v1.1.1/docs/clickhouse/getting-started/#clickhouse-backend"
},"117": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "How to run TPC-DS",
- "content": "wait to add . ",
- "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-ds",
+ "doc": "v1.1.1/Getting Started with ClickHouse Backend",
+ "title": "v1.1.1/Getting Started with ClickHouse Backend",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/clickhouse/getting-started/",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-ds"
+ "relUrl": "/archives/v1.1.1/docs/clickhouse/getting-started/"
},"118": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "How to track the memory exhaust problem",
- "content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
- "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Supported Version",
+ "content": "| Type | Version |——-|——————————| Spark | 3.2.2, 3.3.1 | OS | Ubuntu20.04/22.04, Centos7/8 | jdk | openjdk8 | scala | 2.12 . Spark3.4.0 support is still WIP. TPCH/DS can pass, UT is not yet passed. There are pending PRs for jdk11 support. Currently, the mvn script can automatically fetch and build all dependency libraries incluing Velox. Our nightly build still use Velox under oap-project. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#supported-version",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#supported-version"
},"119": {
- "doc": "How To Use Gluten(v1.2.1)",
- "title": "How To Use Gluten(v1.2.1)",
- "content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
- "url": "/archives/v1.2.1/docs/developers/how-to-use/",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Prerequisite",
+ "content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Gluten builds with Spark3.2.x and Spark3.3.x now but only fully tested in CI with 3.2.2 and 3.3.1. We will add/update supported/tested versions according to the upstream changes. we need to set up the JAVA_HOME env. Currently, java 8 is required and the support for j [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#prerequisite",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#prerequisite"
},"120": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "How to understand the key work of Gluten?",
- "content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
- "url": "/archives/v1.1.1/developers/howto/#how-to-understand-the-key-work-of-gluten",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Build Gluten with Velox Backend",
+ "content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . cd /path/to/gluten ## The script builds two jars for spark 3.2.2 and 3.3.1./dev/buildbundle-veloxbe.sh ## After a complete build, if you need to re-build the project and only some gluten code is changed, ## you can use the following command to skip building velox and protobuf. [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-velox-backend",
- "relUrl": "/archives/v1.1.1/developers/howto/#how-to-understand-the-key-work-of-gluten"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-velox-backend"
},"121": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "How to debug in Gluten?",
- "content": " ",
- "url": "/archives/v1.1.1/developers/howto/#how-to-debug-in-gluten",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Dependency library deployment",
+ "content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#dependency-library-deployment",
- "relUrl": "/archives/v1.1.1/developers/howto/#how-to-debug-in-gluten"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#dependency-library-deployment"
},"122": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "1 How to debug C++",
- "content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
- "url": "/archives/v1.1.1/developers/howto/#1-how-to-debug-c",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "HDFS support",
+ "content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#hdfs-support",
- "relUrl": "/archives/v1.1.1/developers/howto/#1-how-to-debug-c"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#hdfs-support"
},"123": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "2 How to debug plan validation process",
- "content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
- "url": "/archives/v1.1.1/developers/howto/#2-how-to-debug-plan-validation-process",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "AWS S3 support",
+ "content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#aws-s3-support",
- "relUrl": "/archives/v1.1.1/developers/howto/#2-how-to-debug-plan-validation-process"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#aws-s3-support"
},"124": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "3 How to debug Java/Scala",
- "content": "wait to add . ",
- "url": "/archives/v1.1.1/developers/howto/#3-how-to-debug-javascala",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Celeborn support",
+ "content": "Gluten with velox backend supports Celeborn as remote shuffle service. Below introduction is used to enable this feature . First refer to this URL(https://github.com/apache/incubator-celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable rss profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Prss -DskipTests . Then add the Gluten and Spark Celeborn Client packages to your Spark application’s classpath(usua [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#celeborn-support",
- "relUrl": "/archives/v1.1.1/developers/howto/#3-how-to-debug-javascala"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#celeborn-support"
},"125": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "4 How to debug with core-dump",
- "content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
- "url": "/archives/v1.1.1/developers/howto/#4-how-to-debug-with-core-dump",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "DeltaLake Support",
+ "content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with velox backends [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#deltalake-support",
- "relUrl": "/archives/v1.1.1/developers/howto/#4-how-to-debug-with-core-dump"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#deltalake-support"
},"126": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "How to run TPC-H on Velox backend",
- "content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox-be by workflow, you can refer to velox_be.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, prepare the d [...]
- "url": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-h-on-velox-backend",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Iceberg Support",
+ "content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Then, put the additional gluten-iceberg jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-iceberg jar is in gluten-iceberg/target directory. After the two steps, you can query icebe [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#iceberg-support",
- "relUrl": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-h-on-velox-backend"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#iceberg-support"
},"127": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "How to run TPC-DS",
- "content": "wait to add . ",
- "url": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-ds",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Coverage",
+ "content": "Spark3.3 has 387 functions in total. ~240 are commonly used. Velox’s functions have two category, Presto and Spark. Presto has 124 functions implemented. Spark has 62 functions. Spark functions are verified to have the same result as Vanilla Spark. Some Presto functions have the same result as Vanilla Spark but some others have different. Gluten prefer to use Spark functions firstly. If it’s not in Spark’s list but implemented in Presto, we currently offload to Presto one [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#coverage",
- "relUrl": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-ds"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#coverage"
},"128": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "How to track the memory exhaust problem",
- "content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
- "url": "/archives/v1.1.1/developers/howto/#how-to-track-the-memory-exhaust-problem",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Spill (Experimental)",
+ "content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#spill-experimental",
- "relUrl": "/archives/v1.1.1/developers/howto/#how-to-track-the-memory-exhaust-problem"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#spill-experimental"
},"129": {
- "doc": "v1.1.1/How To Use Gluten",
- "title": "v1.1.1/How To Use Gluten",
- "content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
- "url": "/archives/v1.1.1/developers/howto/",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Velox User-Defined Functions (UDF)",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#velox-user-defined-functions-udf",
- "relUrl": "/archives/v1.1.1/developers/howto/"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#velox-user-defined-functions-udf"
},"130": {
- "doc": "How To Use Gluten",
- "title": "How to understand the key work of Gluten?",
- "content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
- "url": "/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Introduction",
+ "content": "Velox backend supports User-Defined Functions (UDF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#introduction",
- "relUrl": "/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#introduction"
},"131": {
- "doc": "How To Use Gluten",
- "title": "How to debug in Gluten?",
- "content": " ",
- "url": "/docs/developers/how-to-use/#how-to-debug-in-gluten",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Creating a UDF library",
+ "content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#creating-a-udf-library",
- "relUrl": "/docs/developers/how-to-use/#how-to-debug-in-gluten"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#creating-a-udf-library"
},"132": {
- "doc": "How To Use Gluten",
- "title": "1 How to debug C++",
- "content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
- "url": "/docs/developers/how-to-use/#1-how-to-debug-c",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Building the UDF library",
+ "content": "To build the UDF library, users need to compile the C++ code and link to libvelox.so. It’s recommended to create a CMakeLists.txt for the project. Here’s an example: . project(myudf) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(GLUTEN_HOME /path/to/gluten) add_library(myudf SHARED \"MyUDF.cpp\") find_library(VELOX_LIBRARY REQUIRED NAMES velox HINTS ${GLUTEN_HOME}/cpp/build/releases NO_DEFAULT_PATH) target_include_directories(myudf PRIVATE ${GLUTEN_HO [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#building-the-udf-library",
- "relUrl": "/docs/developers/how-to-use/#1-how-to-debug-c"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#building-the-udf-library"
},"133": {
- "doc": "How To Use Gluten",
- "title": "2 How to debug plan validation process",
- "content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
- "url": "/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Using UDF in Gluten",
+ "content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the libray paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#using-udf-in-gluten",
- "relUrl": "/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#using-udf-in-gluten"
},"134": {
- "doc": "How To Use Gluten",
- "title": "3 How to debug Java/Scala",
- "content": "wait to add . ",
- "url": "/docs/developers/how-to-use/#3-how-to-debug-javascala",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Try the example",
+ "content": "We provided an Velox UDF example file MyUDF.cpp. After building gluten cpp, you can find the example library at /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so . Start spark-shell or spark-sql with below configuration . --files /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so --conf spark.gluten.sql.columnar.backend.velox.udfLibraryPaths=libmyudf.so . Run query. The functions myudf1 and myudf2 increment the input value by a constant of 5 . select myudf1(1 [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#try-the-example",
- "relUrl": "/docs/developers/how-to-use/#3-how-to-debug-javascala"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#try-the-example"
},"135": {
- "doc": "How To Use Gluten",
- "title": "4 How to debug with core-dump",
- "content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
- "url": "/docs/developers/how-to-use/#4-how-to-debug-with-core-dump",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "High-Bandwidth Memory (HBM) support",
+ "content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#high-bandwidth-memory-hbm-support",
- "relUrl": "/docs/developers/how-to-use/#4-how-to-debug-with-core-dump"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#high-bandwidth-memory-hbm-support"
},"136": {
- "doc": "How To Use Gluten",
- "title": "How to run TPC-H on Velox backend",
- "content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox with docker by workflow, you can refer to velox_docker.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, [...]
- "url": "/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Build Gluten with HBM",
+ "content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds two jars for spark 3.2.2 and 3.3.1./dev/buildbundle-veloxbe.sh --enable_hbm=ON . ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-hbm",
- "relUrl": "/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-hbm"
},"137": {
- "doc": "How To Use Gluten",
- "title": "How to run TPC-DS",
- "content": "wait to add . ",
- "url": "/docs/developers/how-to-use/#how-to-run-tpc-ds",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Configure and enable HBM in Spark Application",
+ "content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#configure-and-enable-hbm-in-spark-application",
- "relUrl": "/docs/developers/how-to-use/#how-to-run-tpc-ds"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#configure-and-enable-hbm-in-spark-application"
},"138": {
- "doc": "How To Use Gluten",
- "title": "How to track the memory exhaust problem",
- "content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
- "url": "/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Intel® QuickAssist Technology (QAT) support",
+ "content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#intel-quickassist-technology-qat-support",
- "relUrl": "/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#intel-quickassist-technology-qat-support"
},"139": {
- "doc": "How To Use Gluten",
- "title": "How To Use Gluten",
- "content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
- "url": "/docs/developers/how-to-use/",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Software Requirements",
+ "content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#software-requirements",
- "relUrl": "/docs/developers/how-to-use/"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#software-requirements"
},"140": {
- "doc": "How To Release(v1.2.1)",
- "title": "How to Release",
- "content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
- "url": "/archives/v1.2.1/docs/developers/how-to-release/#how-to-release",
-
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-release/#how-to-release"
- },"141": {
- "doc": "How To Release(v1.2.1)",
- "title": "Release Process",
- "content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
- "url": "/archives/v1.2.1/docs/developers/how-to-release/#release-process",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Build Gluten with QAT",
+ "content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-qat",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-release/#release-process"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-qat"
+ },"141": {
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Enable QAT with Gzip/Zstd for shuffle compression",
+ "content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#enable-qat-with-gzipzstd-for-shuffle-compression",
+
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#enable-qat-with-gzipzstd-for-shuffle-compression"
},"142": {
- "doc": "How To Release(v1.2.1)",
- "title": "How To Release(v1.2.1)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/developers/how-to-release/",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "QAT driver references",
+ "content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#qat-driver-references",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-release/"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#qat-driver-references"
},"143": {
- "doc": "v1.1.1/How To Release",
- "title": "How to Release",
- "content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
- "url": "/archives/v1.1.1/developers/howtorelease/#how-to-release",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
+ "content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#intel-in-memory-analytics-accelerator-iaaiax-support",
- "relUrl": "/archives/v1.1.1/developers/howtorelease/#how-to-release"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#intel-in-memory-analytics-accelerator-iaaiax-support"
},"144": {
- "doc": "v1.1.1/How To Release",
- "title": "Release Process",
- "content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
- "url": "/archives/v1.1.1/developers/howtorelease/#release-process",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Build Gluten with IAA",
+ "content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-iaa",
- "relUrl": "/archives/v1.1.1/developers/howtorelease/#release-process"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-iaa"
},"145": {
- "doc": "v1.1.1/How To Release",
- "title": "v1.1.1/How To Release",
- "content": " ",
- "url": "/archives/v1.1.1/developers/howtorelease/",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Enable IAA with Gzip Compression for shuffle compression",
+ "content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#enable-iaa-with-gzip-compression-for-shuffle-compression",
- "relUrl": "/archives/v1.1.1/developers/howtorelease/"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#enable-iaa-with-gzip-compression-for-shuffle-compression"
},"146": {
- "doc": "How To Release",
- "title": "How to Release",
- "content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
- "url": "/docs/developers/how-to-release/#how-to-release",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "IAA references",
+ "content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#iaa-references",
- "relUrl": "/docs/developers/how-to-release/#how-to-release"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#iaa-references"
},"147": {
- "doc": "How To Release",
- "title": "Release Process",
- "content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
- "url": "/docs/developers/how-to-release/#release-process",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
+ "content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
- "relUrl": "/docs/developers/how-to-release/#release-process"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
},"148": {
- "doc": "How To Release",
- "title": "How To Release",
- "content": " ",
- "url": "/docs/developers/how-to-release/",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Data preparation",
+ "content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#data-preparation",
- "relUrl": "/docs/developers/how-to-release/"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#data-preparation"
},"149": {
- "doc": "How To Scan Security issues(v1.2.1)",
- "title": "How to scan the security issues",
- "content": "This section outlines the steps to use tools to scan Apache Gluten (incubating) source code and make sure no vulnerability issues in the code. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
- "url": "/archives/v1.2.1/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Submit the Spark SQL job",
+ "content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#submit-the-spark-sql-job",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#submit-the-spark-sql-job"
},"150": {
- "doc": "How To Scan Security issues(v1.2.1)",
- "title": "Scan Security Process",
- "content": "Before every Apache Gluten (incubating) release, we need to ensure there is no vulnerability issue in the source code. We use Trivy as the tool to scan all the security issues. | Install Trivy, please follow the steps to install Trivy: Trivy Installation . | Configuring Trivy, please follow the guide to configure Trivy for specific operation: Trivy Configuration . | Run Trivy File System Scan with the source code. Below is an example about how we run Trivy scan with Apach [...]
- "url": "/archives/v1.2.1/docs/developers/how-to-security-scan/#scan-security-process",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Result",
+ "content": "wholestagetransformer indicates that the offload works. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#result",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/#scan-security-process"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#result"
},"151": {
- "doc": "How To Scan Security issues(v1.2.1)",
- "title": "How To Scan Security issues(v1.2.1)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/developers/how-to-security-scan/",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Performance",
+ "content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#performance",
- "relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#performance"
},"152": {
- "doc": "How To Scan Security issues",
- "title": "How to scan the security issues",
- "content": "This section outlines the steps to use tools to scan Apache Gluten (incubating) source code and make sure no vulnerability issues in the code. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
- "url": "/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "External reference setup",
+ "content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#external-reference-setup",
- "relUrl": "/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#external-reference-setup"
},"153": {
- "doc": "How To Scan Security issues",
- "title": "Scan Security Process",
- "content": "Before every Apache Gluten (incubating) release, we need to ensure there is no vulnerability issue in the source code. We use Trivy as the tool to scan all the security issues. | Install Trivy, please follow the steps to install Trivy: Trivy Installation . | Configuring Trivy, please follow the guide to configure Trivy for specific operation: Trivy Configuration . | Run Trivy File System Scan with the source code. Below is an example about how we run Trivy scan with Apach [...]
- "url": "/docs/developers/how-to-security-scan/#scan-security-process",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Gluten UI",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-ui",
- "relUrl": "/docs/developers/how-to-security-scan/#scan-security-process"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-ui"
},"154": {
- "doc": "How To Scan Security issues",
- "title": "How To Scan Security issues",
- "content": " ",
- "url": "/docs/developers/how-to-security-scan/",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Gluten event",
+ "content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-event",
- "relUrl": "/docs/developers/how-to-security-scan/"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-event"
},"155": {
- "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
- "title": "Generate Micro Benchmarks for Velox Backend",
- "content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
- "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "SQL tab",
+ "content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#sql-tab",
- "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#sql-tab"
},"156": {
- "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
- "title": "Try the example",
- "content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
- "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#try-the-example",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "History server",
+ "content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#history-server",
- "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#try-the-example"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#history-server"
},"157": {
- "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
- "title": "Generate Substrait plan and input for any query",
- "content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
- "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Native plan string",
+ "content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-string",
- "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-string"
},"158": {
- "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
- "title": "Save ouput to parquet to analyze",
- "content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
- "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Native plan with stats",
+ "content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-with-stats",
- "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-with-stats"
},"159": {
- "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
- "title": "Add shuffle write process",
- "content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
- "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#add-shuffle-write-process",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "Gluten Implicits",
+ "content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-implicits",
- "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#add-shuffle-write-process"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-implicits"
},"160": {
- "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
- "title": "Simulate Spark with multiple processes and threads",
- "content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
- "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads",
+ "doc": "v1.1.1/Getting-Started with Velox Backend",
+ "title": "v1.1.1/Getting-Started with Velox Backend",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/velox/getting-started/",
- "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads"
+ "relUrl": "/archives/v1.1.1/docs/velox/getting-started/"
},"161": {
- "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
- "title": "Micro Benchmarks for Velox Backend(v1.2.1)",
+ "doc": "v1.1.1/Velox Backend",
+ "title": "Gluten Velox Backend Documents",
"content": " ",
- "url": "/archives/v1.2.1/docs/developers/microbenchmarks/",
+ "url": "/archives/v1.1.1/docs/velox/#gluten-velox-backend-documents",
- "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/"
+ "relUrl": "/archives/v1.1.1/docs/velox/#gluten-velox-backend-documents"
},"162": {
- "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
- "title": "Generate Micro Benchmarks for Velox Backend",
- "content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
- "url": "/archives/v1.1.1/developers/microbenchmark/#generate-micro-benchmarks-for-velox-backend",
+ "doc": "v1.1.1/Velox Backend",
+ "title": "v1.1.1/Velox Backend",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/velox/",
- "relUrl": "/archives/v1.1.1/developers/microbenchmark/#generate-micro-benchmarks-for-velox-backend"
+ "relUrl": "/archives/v1.1.1/docs/velox/"
},"163": {
- "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
- "title": "Try the example",
- "content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
- "url": "/archives/v1.1.1/developers/microbenchmark/#try-the-example",
+ "doc": "v1.1.1/Build Parameters for Velox Backend",
+ "title": "Build Parameters",
+ "content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | ON | . | build_protob [...]
+ "url": "/archives/v1.1.1/docs/velox/build_parameters/#build-parameters",
- "relUrl": "/archives/v1.1.1/developers/microbenchmark/#try-the-example"
+ "relUrl": "/archives/v1.1.1/docs/velox/build_parameters/#build-parameters"
},"164": {
- "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
- "title": "Generate Substrait plan and input for any query",
- "content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
- "url": "/archives/v1.1.1/developers/microbenchmark/#generate-substrait-plan-and-input-for-any-query",
+ "doc": "v1.1.1/Build Parameters for Velox Backend",
+ "title": "Gluten Jar for Deployment",
+ "content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . ",
+ "url": "/archives/v1.1.1/docs/velox/build_parameters/#gluten-jar-for-deployment",
- "relUrl": "/archives/v1.1.1/developers/microbenchmark/#generate-substrait-plan-and-input-for-any-query"
+ "relUrl": "/archives/v1.1.1/docs/velox/build_parameters/#gluten-jar-for-deployment"
},"165": {
- "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
- "title": "Save ouput to parquet to analyze",
- "content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
- "url": "/archives/v1.1.1/developers/microbenchmark/#save-ouput-to-parquet-to-analyze",
+ "doc": "v1.1.1/Build Parameters for Velox Backend",
+ "title": "v1.1.1/Build Parameters for Velox Backend",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/velox/build_parameters/",
- "relUrl": "/archives/v1.1.1/developers/microbenchmark/#save-ouput-to-parquet-to-analyze"
+ "relUrl": "/archives/v1.1.1/docs/velox/build_parameters/"
},"166": {
- "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
- "title": "Add shuffle write process",
- "content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
- "url": "/archives/v1.1.1/developers/microbenchmark/#add-shuffle-write-process",
-
- "relUrl": "/archives/v1.1.1/developers/microbenchmark/#add-shuffle-write-process"
- },"167": {
- "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
- "title": "Simulate Spark with multiple processes and threads",
- "content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
- "url": "/archives/v1.1.1/developers/microbenchmark/#simulate-spark-with-multiple-processes-and-threads",
+ "doc": "v1.1.1/Using GCS with Gluten",
+ "title": "Working with GCS",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/velox/gcs/#working-with-gcs",
- "relUrl": "/archives/v1.1.1/developers/microbenchmark/#simulate-spark-with-multiple-processes-and-threads"
+ "relUrl": "/archives/v1.1.1/docs/velox/gcs/#working-with-gcs"
+ },"167": {
+ "doc": "v1.1.1/Using GCS with Gluten",
+ "title": "Installing the gcloud CLI",
+ "content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
+ "url": "/archives/v1.1.1/docs/velox/gcs/#installing-the-gcloud-cli",
+
+ "relUrl": "/archives/v1.1.1/docs/velox/gcs/#installing-the-gcloud-cli"
},"168": {
- "doc": "v1.1.1/Micro Benchmarks for Velox Backend",
- "title": "v1.1.1/Micro Benchmarks for Velox Backend",
- "content": " ",
- "url": "/archives/v1.1.1/developers/microbenchmark/",
+ "doc": "v1.1.1/Using GCS with Gluten",
+ "title": "Configuring GCS using a user account",
+ "content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
+ "url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-user-account",
- "relUrl": "/archives/v1.1.1/developers/microbenchmark/"
+ "relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-user-account"
},"169": {
- "doc": "Micro Benchmarks for Velox Backend",
- "title": "Generate Micro Benchmarks for Velox Backend",
- "content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
- "url": "/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend",
+ "doc": "v1.1.1/Using GCS with Gluten",
+ "title": "Configuring GCS using a credential file",
+ "content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
+ "url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-credential-file",
- "relUrl": "/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend"
+ "relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-credential-file"
},"170": {
- "doc": "Micro Benchmarks for Velox Backend",
- "title": "Try the example",
- "content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
- "url": "/docs/developers/microbenchmarks/#try-the-example",
+ "doc": "v1.1.1/Using GCS with Gluten",
+ "title": "Configuring GCS endpoints",
+ "content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
+ "url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-endpoints",
- "relUrl": "/docs/developers/microbenchmarks/#try-the-example"
+ "relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-endpoints"
},"171": {
- "doc": "Micro Benchmarks for Velox Backend",
- "title": "Generate Substrait plan and input for any query",
- "content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
- "url": "/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query",
+ "doc": "v1.1.1/Using GCS with Gluten",
+ "title": "v1.1.1/Using GCS with Gluten",
+ "content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
+ "url": "/archives/v1.1.1/docs/velox/gcs/",
- "relUrl": "/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query"
+ "relUrl": "/archives/v1.1.1/docs/velox/gcs/"
},"172": {
- "doc": "Micro Benchmarks for Velox Backend",
- "title": "Save ouput to parquet to analyze",
- "content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
- "url": "/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze",
+ "doc": "v1.1.1/Using S3 with Gluten",
+ "title": "Working with S3",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/velox/s3/#working-with-s3",
- "relUrl": "/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze"
+ "relUrl": "/archives/v1.1.1/docs/velox/s3/#working-with-s3"
},"173": {
- "doc": "Micro Benchmarks for Velox Backend",
- "title": "Add shuffle write process",
- "content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
- "url": "/docs/developers/microbenchmarks/#add-shuffle-write-process",
+ "doc": "v1.1.1/Using S3 with Gluten",
+ "title": "Configuring S3 endpoint",
+ "content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
+ "url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-endpoint",
- "relUrl": "/docs/developers/microbenchmarks/#add-shuffle-write-process"
+ "relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-endpoint"
},"174": {
- "doc": "Micro Benchmarks for Velox Backend",
- "title": "Simulate Spark with multiple processes and threads",
- "content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
- "url": "/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads",
+ "doc": "v1.1.1/Using S3 with Gluten",
+ "title": "Configuring S3 instance credentials",
+ "content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
+ "url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-instance-credentials",
- "relUrl": "/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads"
+ "relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-instance-credentials"
},"175": {
- "doc": "Micro Benchmarks for Velox Backend",
- "title": "Micro Benchmarks for Velox Backend",
- "content": " ",
- "url": "/docs/developers/microbenchmarks/",
+ "doc": "v1.1.1/Using S3 with Gluten",
+ "title": "Configuring S3 IAM roles",
+ "content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
+ "url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-iam-roles",
- "relUrl": "/docs/developers/microbenchmarks/"
+ "relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-iam-roles"
},"176": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Environment",
- "content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
- "url": "/archives/v1.2.1/docs/developers/new-to/#environment",
+ "doc": "v1.1.1/Using S3 with Gluten",
+ "title": "Other authentatication methods are not supported yet",
+ "content": ". | AWS temporary credential | . ",
+ "url": "/archives/v1.1.1/docs/velox/s3/#other-authentatication-methods-are-not-supported-yet",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#environment"
+ "relUrl": "/archives/v1.1.1/docs/velox/s3/#other-authentatication-methods-are-not-supported-yet"
},"177": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Openjdk8",
- "content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#openjdk8",
+ "doc": "v1.1.1/Using S3 with Gluten",
+ "title": "Log granularity of AWS C++ SDK in velox",
+ "content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
+ "url": "/archives/v1.1.1/docs/velox/s3/#log-granularity-of-aws-c-sdk-in-velox",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#openjdk8"
+ "relUrl": "/archives/v1.1.1/docs/velox/s3/#log-granularity-of-aws-c-sdk-in-velox"
},"178": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Maven 3.6.3 or above",
- "content": "Maven Dowload Page And then set the environment setting. ",
- "url": "/archives/v1.2.1/docs/developers/new-to/#maven-363-or-above",
+ "doc": "v1.1.1/Using S3 with Gluten",
+ "title": "Local Caching support",
+ "content": "Velox supports a local cache when reading data from HDFS/S3. The feature is very useful if remote storage is slow, e.g., reading from a public S3 bucket and stronger performance is desired. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage, and the future reading requests on already cached blocks will be serviced from local cache files. To enable the local caching feature, below configurations are required: . spark.g [...]
+ "url": "/archives/v1.1.1/docs/velox/s3/#local-caching-support",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#maven-363-or-above"
+ "relUrl": "/archives/v1.1.1/docs/velox/s3/#local-caching-support"
},"179": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "GCC 9.4 or above",
- "content": " ",
- "url": "/archives/v1.2.1/docs/developers/new-to/#gcc-94-or-above",
+ "doc": "v1.1.1/Using S3 with Gluten",
+ "title": "v1.1.1/Using S3 with Gluten",
+ "content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
+ "url": "/archives/v1.1.1/docs/velox/s3/",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#gcc-94-or-above"
+ "relUrl": "/archives/v1.1.1/docs/velox/s3/"
},"180": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Compile gluten using debug mode",
- "content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#compile-gluten-using-debug-mode",
+ "doc": "v1.1.1/Documentation",
+ "title": "Gluten Documents by version",
+ "content": " ",
+ "url": "/archives/v1.1.1/docs/#gluten-documents-by-version",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#compile-gluten-using-debug-mode"
+ "relUrl": "/archives/v1.1.1/docs/#gluten-documents-by-version"
},"181": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Java/scala code development with Intellij",
+ "doc": "v1.1.1/Documentation",
+ "title": "v1.1.1/Documentation",
"content": " ",
- "url": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-development-with-intellij",
+ "url": "/archives/v1.1.1/docs/",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-development-with-intellij"
+ "relUrl": "/archives/v1.1.1/docs/"
},"182": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Linux intellij local debug",
- "content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#linux-intellij-local-debug",
+ "doc": "v1.1.1",
+ "title": "Gluten Documents by version",
+ "content": " ",
+ "url": "/archives/v1.1.1/#gluten-documents-by-version",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#linux-intellij-local-debug"
+ "relUrl": "/archives/v1.1.1/#gluten-documents-by-version"
},"183": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Windows/Mac intellij remote debug",
- "content": "If you have Ultimate intellij, you can try to debug remotely. ",
- "url": "/archives/v1.2.1/docs/developers/new-to/#windowsmac-intellij-remote-debug",
+ "doc": "v1.1.1",
+ "title": "v1.1.1",
+ "content": " ",
+ "url": "/archives/v1.1.1/",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#windowsmac-intellij-remote-debug"
+ "relUrl": "/archives/v1.1.1/"
},"184": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Set up gluten project",
- "content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
- "url": "/archives/v1.2.1/docs/developers/new-to/#set-up-gluten-project",
+ "doc": "Configuration(v1.2.1)",
+ "title": "Spark Configurations for Gluten Plugin",
+ "content": "There are many configurations could impact the Gluten Plugin performance and can be fine-tuned in Spark. You can add these configurations into spark-defaults.conf to enable or disable the setting. ",
+ "url": "/archives/v1.2.1/docs/configuration/#spark-configurations-for-gluten-plugin",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#set-up-gluten-project"
+ "relUrl": "/archives/v1.2.1/docs/configuration/#spark-configurations-for-gluten-plugin"
},"185": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Java/Scala code style",
- "content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-style",
+ "doc": "Configuration(v1.2.1)",
+ "title": "Spark parameters",
+ "content": "| Parameters | Description | Recommend Setting | . | spark.driver.extraClassPath | To add Gluten Plugin jar file in Spark Driver | /path/to/jar_file | . | spark.executor.extraClassPath | To add Gluten Plugin jar file in Spark Executor | /path/to/jar_file | . | spark.executor.memory | To set up how much memory to be used for Spark Executor. | | . | spark.memory.offHeap.size | To set up how much memory to be used for Java OffHeap. Please notice Gluten Plugin will leverage [...]
+ "url": "/archives/v1.2.1/docs/configuration/#spark-parameters",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-style"
+ "relUrl": "/archives/v1.2.1/docs/configuration/#spark-parameters"
},"186": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "CPP code development with Visual Studio Code",
- "content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#cpp-code-development-with-visual-studio-code",
+ "doc": "Configuration(v1.2.1)",
+ "title": "Velox Parameters",
+ "content": "The following configurations are related to Velox settings. | Parameters | Description | Recommend Setting | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The m [...]
+ "url": "/archives/v1.2.1/docs/configuration/#velox-parameters",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#cpp-code-development-with-visual-studio-code"
+ "relUrl": "/archives/v1.2.1/docs/configuration/#velox-parameters"
},"187": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Usage",
- "content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#usage",
+ "doc": "Configuration(v1.2.1)",
+ "title": "Configuration(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/configuration/",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#usage"
+ "relUrl": "/archives/v1.2.1/docs/configuration/"
},"188": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Usefule notes",
- "content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#usefule-notes",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Gluten CPP Core Guidelines",
+ "content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#usefule-notes"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines"
},"189": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Debug cpp code with coredump",
- "content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-code-with-coredump",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Philosophy",
+ "content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#philosophy",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-code-with-coredump"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#philosophy"
},"190": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Debug cpp with gdb",
- "content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-with-gdb",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Code Formatting",
+ "content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-formatting",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-with-gdb"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-formatting"
},"191": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Run TPC-H and TPC-DS",
- "content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_docker.yml . ",
- "url": "/archives/v1.2.1/docs/developers/new-to/#run-tpc-h-and-tpc-ds",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Naming Conventions",
+ "content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#naming-conventions",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#run-tpc-h-and-tpc-ds"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#naming-conventions"
},"192": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "Run gluten+velox on clean machine",
- "content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
- "url": "/archives/v1.2.1/docs/developers/new-to/#run-glutenvelox-on-clean-machine",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Designs",
+ "content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#designs",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/#run-glutenvelox-on-clean-machine"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#designs"
},"193": {
- "doc": "New To Gluten(v1.2.1)",
- "title": "New To Gluten(v1.2.1)",
- "content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
- "url": "/archives/v1.2.1/docs/developers/new-to/",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Source File & Header File",
+ "content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#source-file--header-file",
- "relUrl": "/archives/v1.2.1/docs/developers/new-to/"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#source-file--header-file"
},"194": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Environment",
- "content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
- "url": "/archives/v1.1.1/developers/newto/#environment",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Class",
+ "content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#class",
- "relUrl": "/archives/v1.1.1/developers/newto/#environment"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#class"
},"195": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Openjdk8",
- "content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
- "url": "/archives/v1.1.1/developers/newto/#openjdk8",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Function",
+ "content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#function",
- "relUrl": "/archives/v1.1.1/developers/newto/#openjdk8"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#function"
},"196": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Maven 3.6.3 or above",
- "content": "Maven Dowload Page And then set the environment setting. ",
- "url": "/archives/v1.1.1/developers/newto/#maven-363-or-above",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Variable",
+ "content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#variable",
- "relUrl": "/archives/v1.1.1/developers/newto/#maven-363-or-above"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#variable"
},"197": {
- "doc": "v1.1.1/New To Gluten",
- "title": "GCC 9.4 or above",
- "content": " ",
- "url": "/archives/v1.1.1/developers/newto/#gcc-94-or-above",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Constant",
+ "content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#constant",
- "relUrl": "/archives/v1.1.1/developers/newto/#gcc-94-or-above"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#constant"
},"198": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Compile gluten using debug mode",
- "content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
- "url": "/archives/v1.1.1/developers/newto/#compile-gluten-using-debug-mode",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Macro",
+ "content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#macro",
- "relUrl": "/archives/v1.1.1/developers/newto/#compile-gluten-using-debug-mode"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#macro"
},"199": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Java/scala code development with Intellij",
- "content": " ",
- "url": "/archives/v1.1.1/developers/newto/#javascala-code-development-with-intellij",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Namespace",
+ "content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#namespace",
- "relUrl": "/archives/v1.1.1/developers/newto/#javascala-code-development-with-intellij"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#namespace"
},"200": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Linux intellij local debug",
- "content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
- "url": "/archives/v1.1.1/developers/newto/#linux-intellij-local-debug",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Resource Management",
+ "content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#resource-management",
- "relUrl": "/archives/v1.1.1/developers/newto/#linux-intellij-local-debug"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#resource-management"
},"201": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Windows/Mac intellij remote debug",
- "content": "If you have Ultimate intellij, you can try to debug remotely. ",
- "url": "/archives/v1.1.1/developers/newto/#windowsmac-intellij-remote-debug",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Exception",
+ "content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#exception",
- "relUrl": "/archives/v1.1.1/developers/newto/#windowsmac-intellij-remote-debug"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#exception"
},"202": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Set up gluten project",
- "content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
- "url": "/archives/v1.1.1/developers/newto/#set-up-gluten-project",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "Code Comment",
+ "content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-comment",
- "relUrl": "/archives/v1.1.1/developers/newto/#set-up-gluten-project"
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-comment"
},"203": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Java/Scala code style",
- "content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
- "url": "/archives/v1.1.1/developers/newto/#javascala-code-style",
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "References",
+ "content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/#references",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#references"
+ },"204": {
+ "doc": "CPP Code Style(v1.2.1)",
+ "title": "CPP Code Style(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/developers/cpp-code-style/",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/"
+ },"205": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "How to understand the key work of Gluten?",
+ "content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten"
+ },"206": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "How to debug in Gluten?",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-debug-in-gluten",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-debug-in-gluten"
+ },"207": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "1 How to debug C++",
+ "content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/#1-how-to-debug-c",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#1-how-to-debug-c"
+ },"208": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "2 How to debug plan validation process",
+ "content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process"
+ },"209": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "3 How to debug Java/Scala",
+ "content": "wait to add . ",
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/#3-how-to-debug-javascala",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#3-how-to-debug-javascala"
+ },"210": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "4 How to debug with core-dump",
+ "content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/#4-how-to-debug-with-core-dump",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#4-how-to-debug-with-core-dump"
+ },"211": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "How to run TPC-H on Velox backend",
+ "content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox with docker by workflow, you can refer to velox_docker.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, [...]
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend"
+ },"212": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "How to run TPC-DS",
+ "content": "wait to add . ",
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-ds",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-ds"
+ },"213": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "How to track the memory exhaust problem",
+ "content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem"
+ },"214": {
+ "doc": "How To Use Gluten(v1.2.1)",
+ "title": "How To Use Gluten(v1.2.1)",
+ "content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
+ "url": "/archives/v1.2.1/docs/developers/how-to-use/",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-use/"
+ },"215": {
+ "doc": "How To Release(v1.2.1)",
+ "title": "How to Release",
+ "content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
+ "url": "/archives/v1.2.1/docs/developers/how-to-release/#how-to-release",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-release/#how-to-release"
+ },"216": {
+ "doc": "How To Release(v1.2.1)",
+ "title": "Release Process",
+ "content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
+ "url": "/archives/v1.2.1/docs/developers/how-to-release/#release-process",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-release/#release-process"
+ },"217": {
+ "doc": "How To Release(v1.2.1)",
+ "title": "How To Release(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/developers/how-to-release/",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-release/"
+ },"218": {
+ "doc": "How To Scan Security issues(v1.2.1)",
+ "title": "How to scan the security issues",
+ "content": "This section outlines the steps to use tools to scan Apache Gluten (incubating) source code and make sure no vulnerability issues in the code. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
+ "url": "/archives/v1.2.1/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues"
+ },"219": {
+ "doc": "How To Scan Security issues(v1.2.1)",
+ "title": "Scan Security Process",
+ "content": "Before every Apache Gluten (incubating) release, we need to ensure there is no vulnerability issue in the source code. We use Trivy as the tool to scan all the security issues. | Install Trivy, please follow the steps to install Trivy: Trivy Installation . | Configuring Trivy, please follow the guide to configure Trivy for specific operation: Trivy Configuration . | Run Trivy File System Scan with the source code. Below is an example about how we run Trivy scan with Apach [...]
+ "url": "/archives/v1.2.1/docs/developers/how-to-security-scan/#scan-security-process",
- "relUrl": "/archives/v1.1.1/developers/newto/#javascala-code-style"
- },"204": {
- "doc": "v1.1.1/New To Gluten",
- "title": "CPP code development with Visual Studio Code",
- "content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
- "url": "/archives/v1.1.1/developers/newto/#cpp-code-development-with-visual-studio-code",
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/#scan-security-process"
+ },"220": {
+ "doc": "How To Scan Security issues(v1.2.1)",
+ "title": "How To Scan Security issues(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/developers/how-to-security-scan/",
- "relUrl": "/archives/v1.1.1/developers/newto/#cpp-code-development-with-visual-studio-code"
- },"205": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Usage",
- "content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
- "url": "/archives/v1.1.1/developers/newto/#usage",
+ "relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/"
+ },"221": {
+ "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
+ "title": "Generate Micro Benchmarks for Velox Backend",
+ "content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
+ "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend",
- "relUrl": "/archives/v1.1.1/developers/newto/#usage"
- },"206": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Usefule notes",
- "content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
- "url": "/archives/v1.1.1/developers/newto/#usefule-notes",
+ "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend"
+ },"222": {
+ "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
+ "title": "Try the example",
+ "content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
+ "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#try-the-example",
- "relUrl": "/archives/v1.1.1/developers/newto/#usefule-notes"
- },"207": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Debug cpp code with coredump",
- "content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
- "url": "/archives/v1.1.1/developers/newto/#debug-cpp-code-with-coredump",
+ "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#try-the-example"
+ },"223": {
+ "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
+ "title": "Generate Substrait plan and input for any query",
+ "content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
+ "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query",
- "relUrl": "/archives/v1.1.1/developers/newto/#debug-cpp-code-with-coredump"
- },"208": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Debug cpp with gdb",
- "content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
- "url": "/archives/v1.1.1/developers/newto/#debug-cpp-with-gdb",
+ "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query"
+ },"224": {
+ "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
+ "title": "Save ouput to parquet to analyze",
+ "content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
+ "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze",
- "relUrl": "/archives/v1.1.1/developers/newto/#debug-cpp-with-gdb"
- },"209": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Run TPC-H and TPC-DS",
- "content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_be.yml . ",
- "url": "/archives/v1.1.1/developers/newto/#run-tpc-h-and-tpc-ds",
+ "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze"
+ },"225": {
+ "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
+ "title": "Add shuffle write process",
+ "content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
+ "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#add-shuffle-write-process",
- "relUrl": "/archives/v1.1.1/developers/newto/#run-tpc-h-and-tpc-ds"
- },"210": {
- "doc": "v1.1.1/New To Gluten",
- "title": "Run gluten+velox on clean machine",
- "content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
- "url": "/archives/v1.1.1/developers/newto/#run-glutenvelox-on-clean-machine",
+ "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#add-shuffle-write-process"
+ },"226": {
+ "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
+ "title": "Simulate Spark with multiple processes and threads",
+ "content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
+ "url": "/archives/v1.2.1/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads",
- "relUrl": "/archives/v1.1.1/developers/newto/#run-glutenvelox-on-clean-machine"
- },"211": {
- "doc": "v1.1.1/New To Gluten",
- "title": "v1.1.1/New To Gluten",
- "content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
- "url": "/archives/v1.1.1/developers/newto/",
+ "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads"
+ },"227": {
+ "doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
+ "title": "Micro Benchmarks for Velox Backend(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/developers/microbenchmarks/",
- "relUrl": "/archives/v1.1.1/developers/newto/"
- },"212": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/"
+ },"228": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Environment",
"content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
- "url": "/docs/developers/new-to/#environment",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#environment",
- "relUrl": "/docs/developers/new-to/#environment"
- },"213": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#environment"
+ },"229": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Openjdk8",
"content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
- "url": "/docs/developers/new-to/#openjdk8",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#openjdk8",
- "relUrl": "/docs/developers/new-to/#openjdk8"
- },"214": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#openjdk8"
+ },"230": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Maven 3.6.3 or above",
"content": "Maven Dowload Page And then set the environment setting. ",
- "url": "/docs/developers/new-to/#maven-363-or-above",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#maven-363-or-above",
- "relUrl": "/docs/developers/new-to/#maven-363-or-above"
- },"215": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#maven-363-or-above"
+ },"231": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "GCC 9.4 or above",
"content": " ",
- "url": "/docs/developers/new-to/#gcc-94-or-above",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#gcc-94-or-above",
- "relUrl": "/docs/developers/new-to/#gcc-94-or-above"
- },"216": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#gcc-94-or-above"
+ },"232": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Compile gluten using debug mode",
"content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
- "url": "/docs/developers/new-to/#compile-gluten-using-debug-mode",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#compile-gluten-using-debug-mode",
- "relUrl": "/docs/developers/new-to/#compile-gluten-using-debug-mode"
- },"217": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#compile-gluten-using-debug-mode"
+ },"233": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Java/scala code development with Intellij",
"content": " ",
- "url": "/docs/developers/new-to/#javascala-code-development-with-intellij",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-development-with-intellij",
- "relUrl": "/docs/developers/new-to/#javascala-code-development-with-intellij"
- },"218": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-development-with-intellij"
+ },"234": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Linux intellij local debug",
"content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
- "url": "/docs/developers/new-to/#linux-intellij-local-debug",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#linux-intellij-local-debug",
- "relUrl": "/docs/developers/new-to/#linux-intellij-local-debug"
- },"219": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#linux-intellij-local-debug"
+ },"235": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Windows/Mac intellij remote debug",
"content": "If you have Ultimate intellij, you can try to debug remotely. ",
- "url": "/docs/developers/new-to/#windowsmac-intellij-remote-debug",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#windowsmac-intellij-remote-debug",
- "relUrl": "/docs/developers/new-to/#windowsmac-intellij-remote-debug"
- },"220": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#windowsmac-intellij-remote-debug"
+ },"236": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Set up gluten project",
"content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
- "url": "/docs/developers/new-to/#set-up-gluten-project",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#set-up-gluten-project",
- "relUrl": "/docs/developers/new-to/#set-up-gluten-project"
- },"221": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#set-up-gluten-project"
+ },"237": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Java/Scala code style",
"content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
- "url": "/docs/developers/new-to/#javascala-code-style",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-style",
- "relUrl": "/docs/developers/new-to/#javascala-code-style"
- },"222": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-style"
+ },"238": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "CPP code development with Visual Studio Code",
"content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
- "url": "/docs/developers/new-to/#cpp-code-development-with-visual-studio-code",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#cpp-code-development-with-visual-studio-code",
- "relUrl": "/docs/developers/new-to/#cpp-code-development-with-visual-studio-code"
- },"223": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#cpp-code-development-with-visual-studio-code"
+ },"239": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Usage",
"content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
- "url": "/docs/developers/new-to/#usage",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#usage",
- "relUrl": "/docs/developers/new-to/#usage"
- },"224": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#usage"
+ },"240": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Usefule notes",
"content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
- "url": "/docs/developers/new-to/#usefule-notes",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#usefule-notes",
- "relUrl": "/docs/developers/new-to/#usefule-notes"
- },"225": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#usefule-notes"
+ },"241": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Debug cpp code with coredump",
"content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
- "url": "/docs/developers/new-to/#debug-cpp-code-with-coredump",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-code-with-coredump",
- "relUrl": "/docs/developers/new-to/#debug-cpp-code-with-coredump"
- },"226": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-code-with-coredump"
+ },"242": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Debug cpp with gdb",
"content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
- "url": "/docs/developers/new-to/#debug-cpp-with-gdb",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-with-gdb",
- "relUrl": "/docs/developers/new-to/#debug-cpp-with-gdb"
- },"227": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-with-gdb"
+ },"243": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Run TPC-H and TPC-DS",
"content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_docker.yml . ",
- "url": "/docs/developers/new-to/#run-tpc-h-and-tpc-ds",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#run-tpc-h-and-tpc-ds",
- "relUrl": "/docs/developers/new-to/#run-tpc-h-and-tpc-ds"
- },"228": {
- "doc": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#run-tpc-h-and-tpc-ds"
+ },"244": {
+ "doc": "New To Gluten(v1.2.1)",
"title": "Run gluten+velox on clean machine",
"content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
- "url": "/docs/developers/new-to/#run-glutenvelox-on-clean-machine",
+ "url": "/archives/v1.2.1/docs/developers/new-to/#run-glutenvelox-on-clean-machine",
- "relUrl": "/docs/developers/new-to/#run-glutenvelox-on-clean-machine"
- },"229": {
- "doc": "New To Gluten",
- "title": "New To Gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/#run-glutenvelox-on-clean-machine"
+ },"245": {
+ "doc": "New To Gluten(v1.2.1)",
+ "title": "New To Gluten(v1.2.1)",
"content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
- "url": "/docs/developers/new-to/",
+ "url": "/archives/v1.2.1/docs/developers/new-to/",
- "relUrl": "/docs/developers/new-to/"
- },"230": {
+ "relUrl": "/archives/v1.2.1/docs/developers/new-to/"
+ },"246": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Substrait Modifications in Gluten",
"content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
"url": "/archives/v1.2.1/docs/developers/substrait/#substrait-modifications-in-gluten",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/#substrait-modifications-in-gluten"
- },"231": {
+ },"247": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Modifications to algebra.proto",
"content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
"url": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-algebraproto",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-algebraproto"
- },"232": {
+ },"248": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Modifications to type.proto",
"content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
"url": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-typeproto",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-typeproto"
- },"233": {
+ },"249": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Substrait Modifications(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/substrait/",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/"
- },"234": {
- "doc": "v1.1.1/Substrait Modifications",
- "title": "Substrait Modifications in Gluten",
- "content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
- "url": "/archives/v1.1.1/developers/substrait/#substrait-modifications-in-gluten",
+ },"250": {
+ "doc": "Docker script for CentOS 7(v1.2.1)",
+ "title": "Docker script for CentOS 7(v1.2.1)",
+ "content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git236 \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ autoconf \\ autoconf-archive \\ automake \\ perl-IPC-Cmd \\ libicu-devel \\ [...]
+ "url": "/archives/v1.2.1/docs/developers/docker-centos7/",
- "relUrl": "/archives/v1.1.1/developers/substrait/#substrait-modifications-in-gluten"
- },"235": {
- "doc": "v1.1.1/Substrait Modifications",
- "title": "Modifications to algebra.proto",
- "content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
- "url": "/archives/v1.1.1/developers/substrait/#modifications-to-algebraproto",
+ "relUrl": "/archives/v1.2.1/docs/developers/docker-centos7/"
+ },"251": {
+ "doc": "Docker script for CentOS 8(v1.2.1)",
+ "title": "Docker script for CentOS 8(v1.2.1)",
+ "content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
+ "url": "/archives/v1.2.1/docs/developers/docker-centos8/",
- "relUrl": "/archives/v1.1.1/developers/substrait/#modifications-to-algebraproto"
- },"236": {
- "doc": "v1.1.1/Substrait Modifications",
- "title": "Modifications to type.proto",
- "content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
- "url": "/archives/v1.1.1/developers/substrait/#modifications-to-typeproto",
+ "relUrl": "/archives/v1.2.1/docs/developers/docker-centos8/"
+ },"252": {
+ "doc": "Docker script for Ubuntu 22.04/20.04(v1.2.1)",
+ "title": "Docker script for Ubuntu 22.04/20.04(v1.2.1)",
+ "content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
+ "url": "/archives/v1.2.1/docs/developers/docker-ubuntu/",
+
+ "relUrl": "/archives/v1.2.1/docs/developers/docker-ubuntu/"
+ },"253": {
+ "doc": "Developers(v1.2.1)",
+ "title": "Gluten Developers",
+ "content": "This document provides a developer overview of the project and covers the following topics: . ",
+ "url": "/archives/v1.2.1/docs/developers/#gluten-developers",
- "relUrl": "/archives/v1.1.1/developers/substrait/#modifications-to-typeproto"
- },"237": {
- "doc": "v1.1.1/Substrait Modifications",
- "title": "v1.1.1/Substrait Modifications",
+ "relUrl": "/archives/v1.2.1/docs/developers/#gluten-developers"
+ },"254": {
+ "doc": "Developers(v1.2.1)",
+ "title": "Developers(v1.2.1)",
"content": " ",
- "url": "/archives/v1.1.1/developers/substrait/",
+ "url": "/archives/v1.2.1/docs/developers/",
- "relUrl": "/archives/v1.1.1/developers/substrait/"
- },"238": {
- "doc": "Substrait Modifications",
- "title": "Substrait Modifications in Gluten",
- "content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
- "url": "/docs/developers/substrait/#substrait-modifications-in-gluten",
+ "relUrl": "/archives/v1.2.1/docs/developers/"
+ },"255": {
+ "doc": "Velox Function Development(v1.2.1)",
+ "title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
+ "content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
+ "url": "/archives/v1.2.1/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
- "relUrl": "/docs/developers/substrait/#substrait-modifications-in-gluten"
- },"239": {
- "doc": "Substrait Modifications",
- "title": "Modifications to algebra.proto",
- "content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
- "url": "/docs/developers/substrait/#modifications-to-algebraproto",
+ "relUrl": "/archives/v1.2.1/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
+ },"256": {
+ "doc": "Velox Function Development(v1.2.1)",
+ "title": "Velox Function Development(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/developers/velox-function-dev/",
- "relUrl": "/docs/developers/substrait/#modifications-to-algebraproto"
- },"240": {
- "doc": "Substrait Modifications",
- "title": "Modifications to type.proto",
- "content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
- "url": "/docs/developers/substrait/#modifications-to-typeproto",
+ "relUrl": "/archives/v1.2.1/docs/developers/velox-function-dev/"
+ },"257": {
+ "doc": "Getting start with ClickHouse Backend(v1.2.1)",
+ "title": "ClickHouse Backend",
+ "content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
+ "url": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/#clickhouse-backend",
- "relUrl": "/docs/developers/substrait/#modifications-to-typeproto"
- },"241": {
- "doc": "Substrait Modifications",
- "title": "Substrait Modifications",
+ "relUrl": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/#clickhouse-backend"
+ },"258": {
+ "doc": "Getting start with ClickHouse Backend(v1.2.1)",
+ "title": "Getting start with ClickHouse Backend(v1.2.1)",
"content": " ",
- "url": "/docs/developers/substrait/",
+ "url": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/",
- "relUrl": "/docs/developers/substrait/"
- },"242": {
+ "relUrl": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/"
+ },"259": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Supported Version",
"content": "| Type | Version | . | Spark | 3.2.2, 3.3.1, 3.4.2, 3.5.1 | . | OS | Ubuntu20.04/22.04, Centos7/8 | . | jdk | openjdk8/jdk17 | . | scala | 2.12 | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#supported-version",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#supported-version"
- },"243": {
+ },"260": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Prerequisite",
"content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos7/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.2 and 3.5.1. We need to set up the JAVA_HOME env. Currently, Gluten supports java 8 and java 17. For x86_64 . ## make sure jdk8 is used export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#prerequisite",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#prerequisite"
- },"244": {
+ },"261": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with Velox Backend",
"content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . First time build for all supported spark versions./dev/buildbundle-veloxbe.sh . After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and setting up build dependencies./dev/buildbundle-veloxbe.sh --enable_ep [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-velox-backend",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-velox-backend"
- },"245": {
+ },"262": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Dependency library deployment",
"content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#dependency-library-deployment",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#dependency-library-deployment"
- },"246": {
+ },"263": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "HDFS support",
"content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#hdfs-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#hdfs-support"
- },"247": {
+ },"264": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Azure Blob File System (ABFS) support",
"content": "Velox supports ABFS with the open source Azure SDK for C++ and Gluten uses the Velox ABFS connector to connect with ABFS. The build option for ABFS (enable_abfs) must be set to enable this feature as listed below. cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_abfs=ON . Please refer Velox ABFS part for more detailed configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#azure-blob-file-system-abfs-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#azure-blob-file-system-abfs-support"
- },"248": {
+ },"265": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "AWS S3 support",
"content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#aws-s3-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#aws-s3-support"
- },"249": {
+ },"266": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Celeborn support",
"content": "Gluten with velox backend supports Celeborn as remote shuffle service. Currently, the supported Celeborn versions are 0.3.x, 0.4.x and 0.5.0. Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable celeborn profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests . Then add the Gluten and Spa [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#celeborn-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#celeborn-support"
- },"250": {
+ },"267": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Uniffle support",
"content": "Uniffle with velox backend supports Uniffle as remote shuffle service. Currently, the supported Uniffle versions are 0.8.0. First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with uniffle. When compiling the Gluten Java module, it’s required to enable uniffle profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests . Then add the Uniffle and Spark Celeborn Client packages to your Spark application’s classpath(usu [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#uniffle-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#uniffle-support"
- },"251": {
+ },"268": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "DeltaLake Support",
"content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta-XX-SNAPSHOT.jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with vel [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#deltalake-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#deltalake-support"
- },"252": {
+ },"269": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Iceberg Support",
"content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan’s fallback. After the two steps, you can query [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#iceberg-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#iceberg-support"
- },"253": {
+ },"270": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Coverage",
"content": "Spark3.3 has 387 functions in total. ~240 are commonly used. To get the support status of all Spark built-in functions, please refer to Velox Backend’s Supported Operators & Functions. Velox doesn’t support ANSI mode), so as Gluten. Once ANSI mode is enabled in Spark config, Gluten will fallback to Vanilla Spark. To identify what can be offloaded in a query and detailed fallback reasons, user can follow below steps to retrieve corresponding logs. 1) Enable Gluten by p [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#coverage",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#coverage"
- },"254": {
+ },"271": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Spill (Experimental)",
"content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#spill-experimental",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#spill-experimental"
- },"255": {
+ },"272": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": "Please check the VeloxNativeUDF.md for more detailed usage and configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"256": {
+ },"273": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "High-Bandwidth Memory (HBM) support",
"content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#high-bandwidth-memory-hbm-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#high-bandwidth-memory-hbm-support"
- },"257": {
+ },"274": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with HBM",
"content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.2 and 3.5.1./dev/buildbundle-veloxbe.sh -- [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-hbm",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-hbm"
- },"258": {
+ },"275": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Configure and enable HBM in Spark Application",
"content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application"
- },"259": {
+ },"276": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Intel® QuickAssist Technology (QAT) support",
"content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-quickassist-technology-qat-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-quickassist-technology-qat-support"
- },"260": {
+ },"277": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Software Requirements",
"content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#software-requirements",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#software-requirements"
- },"261": {
+ },"278": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with QAT",
"content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-qat",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-qat"
- },"262": {
+ },"279": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Enable QAT with Gzip/Zstd for shuffle compression",
"content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression"
- },"263": {
+ },"280": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "QAT driver references",
"content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#qat-driver-references",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#qat-driver-references"
- },"264": {
+ },"281": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
"content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support"
- },"265": {
+ },"282": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with IAA",
"content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-iaa",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-iaa"
- },"266": {
+ },"283": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Enable IAA with Gzip Compression for shuffle compression",
"content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression"
- },"267": {
+ },"284": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "IAA references",
"content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#iaa-references",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#iaa-references"
- },"268": {
+ },"285": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
"content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
- },"269": {
+ },"286": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Data preparation",
"content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#data-preparation",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#data-preparation"
- },"270": {
+ },"287": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Submit the Spark SQL job",
"content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#submit-the-spark-sql-job",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#submit-the-spark-sql-job"
- },"271": {
+ },"288": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Result",
"content": "wholestagetransformer indicates that the offloading works. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#result",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#result"
- },"272": {
+ },"289": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Performance",
"content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#performance",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#performance"
- },"273": {
+ },"290": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "External reference setup",
"content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#external-reference-setup",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#external-reference-setup"
- },"274": {
+ },"291": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Gluten UI",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-ui",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-ui"
- },"275": {
+ },"292": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Gluten event",
"content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-event",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-event"
- },"276": {
+ },"293": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "SQL tab",
"content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#sql-tab",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#sql-tab"
- },"277": {
+ },"294": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "History server",
"content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#history-server",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#history-server"
- },"278": {
+ },"295": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Native plan string",
"content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-string",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-string"
- },"279": {
+ },"296": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Native plan with stats",
"content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-with-stats",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-with-stats"
- },"280": {
+ },"297": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Gluten Implicits",
"content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-implicits",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-implicits"
- },"281": {
+ },"298": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Getting Start with Velox Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/"
- },"282": {
- "doc": "v1.1.1/Velox Backend",
- "title": "Gluten Velox Backend Documents",
- "content": " ",
- "url": "/archives/v1.1.1/docs/velox/#gluten-velox-backend-documents",
-
- "relUrl": "/archives/v1.1.1/docs/velox/#gluten-velox-backend-documents"
- },"283": {
- "doc": "v1.1.1/Velox Backend",
- "title": "v1.1.1/Velox Backend",
- "content": " ",
- "url": "/archives/v1.1.1/docs/velox/",
-
- "relUrl": "/archives/v1.1.1/docs/velox/"
- },"284": {
- "doc": "Getting Start with Velox Backend",
- "title": "Supported Version",
- "content": "| Type | Version | . | Spark | 3.2.2, 3.3.1, 3.4.2, 3.5.1 | . | OS | Ubuntu20.04/22.04, Centos7/8 | . | jdk | openjdk8/jdk17 | . | scala | 2.12 | . ",
- "url": "/docs/getting-started/velox-backend/#supported-version",
-
- "relUrl": "/docs/getting-started/velox-backend/#supported-version"
- },"285": {
- "doc": "Getting Start with Velox Backend",
- "title": "Prerequisite",
- "content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos7/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.2 and 3.5.1. We need to set up the JAVA_HOME env. Currently, Gluten supports java 8 and java 17. For x86_64 . ## make sure jdk8 is used export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 [...]
- "url": "/docs/getting-started/velox-backend/#prerequisite",
-
- "relUrl": "/docs/getting-started/velox-backend/#prerequisite"
- },"286": {
- "doc": "Getting Start with Velox Backend",
- "title": "Build Gluten with Velox Backend",
- "content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . First time build for all supported spark versions./dev/buildbundle-veloxbe.sh . After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and setting up build dependencies./dev/buildbundle-veloxbe.sh --enable_ep [...]
- "url": "/docs/getting-started/velox-backend/#build-gluten-with-velox-backend",
-
- "relUrl": "/docs/getting-started/velox-backend/#build-gluten-with-velox-backend"
- },"287": {
- "doc": "Getting Start with Velox Backend",
- "title": "Dependency library deployment",
- "content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
- "url": "/docs/getting-started/velox-backend/#dependency-library-deployment",
-
- "relUrl": "/docs/getting-started/velox-backend/#dependency-library-deployment"
- },"288": {
- "doc": "Getting Start with Velox Backend",
- "title": "HDFS support",
- "content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
- "url": "/docs/getting-started/velox-backend/#hdfs-support",
-
- "relUrl": "/docs/getting-started/velox-backend/#hdfs-support"
- },"289": {
- "doc": "Getting Start with Velox Backend",
- "title": "Azure Blob File System (ABFS) support",
- "content": "Velox supports ABFS with the open source Azure SDK for C++ and Gluten uses the Velox ABFS connector to connect with ABFS. The build option for ABFS (enable_abfs) must be set to enable this feature as listed below. cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_abfs=ON . Please refer Velox ABFS part for more detailed configurations. ",
- "url": "/docs/getting-started/velox-backend/#azure-blob-file-system-abfs-support",
-
- "relUrl": "/docs/getting-started/velox-backend/#azure-blob-file-system-abfs-support"
- },"290": {
- "doc": "Getting Start with Velox Backend",
- "title": "AWS S3 support",
- "content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
- "url": "/docs/getting-started/velox-backend/#aws-s3-support",
-
- "relUrl": "/docs/getting-started/velox-backend/#aws-s3-support"
- },"291": {
- "doc": "Getting Start with Velox Backend",
- "title": "Celeborn support",
- "content": "Gluten with velox backend supports Celeborn as remote shuffle service. Currently, the supported Celeborn versions are 0.3.x, 0.4.x and 0.5.0. Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable celeborn profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests . Then add the Gluten and Spa [...]
- "url": "/docs/getting-started/velox-backend/#celeborn-support",
-
- "relUrl": "/docs/getting-started/velox-backend/#celeborn-support"
- },"292": {
- "doc": "Getting Start with Velox Backend",
- "title": "Uniffle support",
- "content": "Uniffle with velox backend supports Uniffle as remote shuffle service. Currently, the supported Uniffle versions are 0.8.0. First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with uniffle. When compiling the Gluten Java module, it’s required to enable uniffle profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests . Then add the Uniffle and Spark Celeborn Client packages to your Spark application’s classpath(usu [...]
- "url": "/docs/getting-started/velox-backend/#uniffle-support",
-
- "relUrl": "/docs/getting-started/velox-backend/#uniffle-support"
- },"293": {
- "doc": "Getting Start with Velox Backend",
- "title": "DeltaLake Support",
- "content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta-XX-SNAPSHOT.jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with vel [...]
- "url": "/docs/getting-started/velox-backend/#deltalake-support",
-
- "relUrl": "/docs/getting-started/velox-backend/#deltalake-support"
- },"294": {
- "doc": "Getting Start with Velox Backend",
- "title": "Iceberg Support",
- "content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan’s fallback. After the two steps, you can query [...]
- "url": "/docs/getting-started/velox-backend/#iceberg-support",
-
- "relUrl": "/docs/getting-started/velox-backend/#iceberg-support"
- },"295": {
- "doc": "Getting Start with Velox Backend",
- "title": "Coverage",
- "content": "Spark3.3 has 387 functions in total. ~240 are commonly used. To get the support status of all Spark built-in functions, please refer to Velox Backend’s Supported Operators & Functions. Velox doesn’t support ANSI mode), so as Gluten. Once ANSI mode is enabled in Spark config, Gluten will fallback to Vanilla Spark. To identify what can be offloaded in a query and detailed fallback reasons, user can follow below steps to retrieve corresponding logs. 1) Enable Gluten by p [...]
- "url": "/docs/getting-started/velox-backend/#coverage",
-
- "relUrl": "/docs/getting-started/velox-backend/#coverage"
- },"296": {
- "doc": "Getting Start with Velox Backend",
- "title": "Spill (Experimental)",
- "content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
- "url": "/docs/getting-started/velox-backend/#spill-experimental",
-
- "relUrl": "/docs/getting-started/velox-backend/#spill-experimental"
- },"297": {
- "doc": "Getting Start with Velox Backend",
- "title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
- "content": "Please check the VeloxNativeUDF.md for more detailed usage and configurations. ",
- "url": "/docs/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
-
- "relUrl": "/docs/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"298": {
- "doc": "Getting Start with Velox Backend",
- "title": "High-Bandwidth Memory (HBM) support",
- "content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
- "url": "/docs/getting-started/velox-backend/#high-bandwidth-memory-hbm-support",
-
- "relUrl": "/docs/getting-started/velox-backend/#high-bandwidth-memory-hbm-support"
},"299": {
- "doc": "Getting Start with Velox Backend",
- "title": "Build Gluten with HBM",
- "content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.2 and 3.5.1./dev/buildbundle-veloxbe.sh -- [...]
- "url": "/docs/getting-started/velox-backend/#build-gluten-with-hbm",
+ "doc": "Using ABFS with Gluten(v1.2.1)",
+ "title": "Working with ABFS",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/getting-started/abfs/#working-with-abfs",
- "relUrl": "/docs/getting-started/velox-backend/#build-gluten-with-hbm"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#working-with-abfs"
},"300": {
- "doc": "Getting Start with Velox Backend",
- "title": "Configure and enable HBM in Spark Application",
- "content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
- "url": "/docs/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application",
+ "doc": "Using ABFS with Gluten(v1.2.1)",
+ "title": "Configuring ABFS Access Token",
+ "content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
+ "url": "/archives/v1.2.1/docs/getting-started/abfs/#configuring-abfs-access-token",
- "relUrl": "/docs/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#configuring-abfs-access-token"
},"301": {
- "doc": "Getting Start with Velox Backend",
- "title": "Intel® QuickAssist Technology (QAT) support",
- "content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
- "url": "/docs/getting-started/velox-backend/#intel-quickassist-technology-qat-support",
+ "doc": "Using ABFS with Gluten(v1.2.1)",
+ "title": "Local Caching support",
+ "content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
+ "url": "/archives/v1.2.1/docs/getting-started/abfs/#local-caching-support",
- "relUrl": "/docs/getting-started/velox-backend/#intel-quickassist-technology-qat-support"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#local-caching-support"
},"302": {
- "doc": "Getting Start with Velox Backend",
- "title": "Software Requirements",
- "content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
- "url": "/docs/getting-started/velox-backend/#software-requirements",
+ "doc": "Using ABFS with Gluten(v1.2.1)",
+ "title": "Using ABFS with Gluten(v1.2.1)",
+ "content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
+ "url": "/archives/v1.2.1/docs/getting-started/abfs/",
- "relUrl": "/docs/getting-started/velox-backend/#software-requirements"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/abfs/"
},"303": {
- "doc": "Getting Start with Velox Backend",
- "title": "Build Gluten with QAT",
- "content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
- "url": "/docs/getting-started/velox-backend/#build-gluten-with-qat",
+ "doc": "Using GCS with Gluten(v1.2.1)",
+ "title": "Working with GCS",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/getting-started/gcs/#working-with-gcs",
- "relUrl": "/docs/getting-started/velox-backend/#build-gluten-with-qat"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#working-with-gcs"
},"304": {
- "doc": "Getting Start with Velox Backend",
- "title": "Enable QAT with Gzip/Zstd for shuffle compression",
- "content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
- "url": "/docs/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression",
+ "doc": "Using GCS with Gluten(v1.2.1)",
+ "title": "Installing the gcloud CLI",
+ "content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
+ "url": "/archives/v1.2.1/docs/getting-started/gcs/#installing-the-gcloud-cli",
- "relUrl": "/docs/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#installing-the-gcloud-cli"
},"305": {
- "doc": "Getting Start with Velox Backend",
- "title": "QAT driver references",
- "content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
- "url": "/docs/getting-started/velox-backend/#qat-driver-references",
+ "doc": "Using GCS with Gluten(v1.2.1)",
+ "title": "Configuring GCS using a user account",
+ "content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
+ "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-user-account",
- "relUrl": "/docs/getting-started/velox-backend/#qat-driver-references"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-user-account"
},"306": {
- "doc": "Getting Start with Velox Backend",
- "title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
- "content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
- "url": "/docs/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support",
+ "doc": "Using GCS with Gluten(v1.2.1)",
+ "title": "Configuring GCS using a credential file",
+ "content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
+ "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file",
- "relUrl": "/docs/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file"
},"307": {
- "doc": "Getting Start with Velox Backend",
- "title": "Build Gluten with IAA",
- "content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
- "url": "/docs/getting-started/velox-backend/#build-gluten-with-iaa",
+ "doc": "Using GCS with Gluten(v1.2.1)",
+ "title": "Configuring GCS endpoints",
+ "content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
+ "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-endpoints",
- "relUrl": "/docs/getting-started/velox-backend/#build-gluten-with-iaa"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-endpoints"
},"308": {
- "doc": "Getting Start with Velox Backend",
- "title": "Enable IAA with Gzip Compression for shuffle compression",
- "content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
- "url": "/docs/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression",
+ "doc": "Using GCS with Gluten(v1.2.1)",
+ "title": "Configuring GCS max retry count",
+ "content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
+ "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-count",
- "relUrl": "/docs/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-count"
},"309": {
- "doc": "Getting Start with Velox Backend",
- "title": "IAA references",
- "content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
- "url": "/docs/getting-started/velox-backend/#iaa-references",
+ "doc": "Using GCS with Gluten(v1.2.1)",
+ "title": "Configuring GCS max retry time",
+ "content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retring (10s, 1m, etc). ",
+ "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-time",
- "relUrl": "/docs/getting-started/velox-backend/#iaa-references"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-time"
},"310": {
- "doc": "Getting Start with Velox Backend",
- "title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
- "content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
- "url": "/docs/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
+ "doc": "Using GCS with Gluten(v1.2.1)",
+ "title": "Using GCS with Gluten(v1.2.1)",
+ "content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
+ "url": "/archives/v1.2.1/docs/getting-started/gcs/",
- "relUrl": "/docs/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/"
},"311": {
- "doc": "Getting Start with Velox Backend",
- "title": "Data preparation",
- "content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
- "url": "/docs/getting-started/velox-backend/#data-preparation",
+ "doc": "Velox Local Caching(v1.2.1)",
+ "title": "Velox Local Caching(v1.2.1)",
+ "content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
+ "url": "/archives/v1.2.1/docs/getting-started/localcache/",
- "relUrl": "/docs/getting-started/velox-backend/#data-preparation"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/localcache/"
},"312": {
- "doc": "Getting Start with Velox Backend",
- "title": "Submit the Spark SQL job",
- "content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
- "url": "/docs/getting-started/velox-backend/#submit-the-spark-sql-job",
+ "doc": "Using S3 with Gluten(v1.2.1)",
+ "title": "Working with S3",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/getting-started/s3/#working-with-s3",
- "relUrl": "/docs/getting-started/velox-backend/#submit-the-spark-sql-job"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#working-with-s3"
},"313": {
- "doc": "Getting Start with Velox Backend",
- "title": "Result",
- "content": "wholestagetransformer indicates that the offloading works. ",
- "url": "/docs/getting-started/velox-backend/#result",
+ "doc": "Using S3 with Gluten(v1.2.1)",
+ "title": "Configuring S3 endpoint",
+ "content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
+ "url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-endpoint",
- "relUrl": "/docs/getting-started/velox-backend/#result"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-endpoint"
},"314": {
- "doc": "Getting Start with Velox Backend",
- "title": "Performance",
- "content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
- "url": "/docs/getting-started/velox-backend/#performance",
+ "doc": "Using S3 with Gluten(v1.2.1)",
+ "title": "Configuring S3 instance credentials",
+ "content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
+ "url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-instance-credentials",
- "relUrl": "/docs/getting-started/velox-backend/#performance"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-instance-credentials"
},"315": {
- "doc": "Getting Start with Velox Backend",
- "title": "External reference setup",
- "content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
- "url": "/docs/getting-started/velox-backend/#external-reference-setup",
+ "doc": "Using S3 with Gluten(v1.2.1)",
+ "title": "Configuring S3 IAM roles",
+ "content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
+ "url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-iam-roles",
- "relUrl": "/docs/getting-started/velox-backend/#external-reference-setup"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-iam-roles"
},"316": {
- "doc": "Getting Start with Velox Backend",
- "title": "Gluten UI",
- "content": " ",
- "url": "/docs/getting-started/velox-backend/#gluten-ui",
+ "doc": "Using S3 with Gluten(v1.2.1)",
+ "title": "Other authentatication methods are not supported yet",
+ "content": ". | AWS temporary credential | . ",
+ "url": "/archives/v1.2.1/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet",
- "relUrl": "/docs/getting-started/velox-backend/#gluten-ui"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet"
},"317": {
- "doc": "Getting Start with Velox Backend",
- "title": "Gluten event",
- "content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
- "url": "/docs/getting-started/velox-backend/#gluten-event",
+ "doc": "Using S3 with Gluten(v1.2.1)",
+ "title": "Log granularity of AWS C++ SDK in velox",
+ "content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
+ "url": "/archives/v1.2.1/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox",
- "relUrl": "/docs/getting-started/velox-backend/#gluten-event"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox"
},"318": {
- "doc": "Getting Start with Velox Backend",
- "title": "SQL tab",
- "content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
- "url": "/docs/getting-started/velox-backend/#sql-tab",
+ "doc": "Using S3 with Gluten(v1.2.1)",
+ "title": "Local Caching support",
+ "content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
+ "url": "/archives/v1.2.1/docs/getting-started/s3/#local-caching-support",
- "relUrl": "/docs/getting-started/velox-backend/#sql-tab"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#local-caching-support"
},"319": {
- "doc": "Getting Start with Velox Backend",
- "title": "History server",
- "content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
- "url": "/docs/getting-started/velox-backend/#history-server",
+ "doc": "Using S3 with Gluten(v1.2.1)",
+ "title": "Using S3 with Gluten(v1.2.1)",
+ "content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
+ "url": "/archives/v1.2.1/docs/getting-started/s3/",
- "relUrl": "/docs/getting-started/velox-backend/#history-server"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/s3/"
},"320": {
- "doc": "Getting Start with Velox Backend",
- "title": "Native plan string",
- "content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
- "url": "/docs/getting-started/velox-backend/#native-plan-string",
+ "doc": "Build Parameters for Velox Backend(v1.2.1)",
+ "title": "Build Parameters",
+ "content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | OFF | . | build_proto [...]
+ "url": "/archives/v1.2.1/docs/getting-started/build-guide/#build-parameters",
- "relUrl": "/docs/getting-started/velox-backend/#native-plan-string"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/#build-parameters"
},"321": {
- "doc": "Getting Start with Velox Backend",
- "title": "Native plan with stats",
- "content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
- "url": "/docs/getting-started/velox-backend/#native-plan-with-stats",
+ "doc": "Build Parameters for Velox Backend(v1.2.1)",
+ "title": "Gluten Jar for Deployment",
+ "content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . | 3.5.1 | 3.5 | 2.12 | . ",
+ "url": "/archives/v1.2.1/docs/getting-started/build-guide/#gluten-jar-for-deployment",
- "relUrl": "/docs/getting-started/velox-backend/#native-plan-with-stats"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/#gluten-jar-for-deployment"
},"322": {
- "doc": "Getting Start with Velox Backend",
- "title": "Gluten Implicits",
- "content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
- "url": "/docs/getting-started/velox-backend/#gluten-implicits",
+ "doc": "Build Parameters for Velox Backend(v1.2.1)",
+ "title": "Build Parameters for Velox Backend(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/getting-started/build-guide/",
- "relUrl": "/docs/getting-started/velox-backend/#gluten-implicits"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/"
},"323": {
- "doc": "Getting Start with Velox Backend",
- "title": "Getting Start with Velox Backend",
+ "doc": "Getting-Started(v1.2.1)",
+ "title": "Getting Started with Gluten for Apache Spark",
"content": " ",
- "url": "/docs/getting-started/velox-backend/",
+ "url": "/archives/v1.2.1/docs/getting-started/#getting-started-with-gluten-for-apache-spark",
- "relUrl": "/docs/getting-started/velox-backend/"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/#getting-started-with-gluten-for-apache-spark"
},"324": {
- "doc": "Using ABFS with Gluten(v1.2.1)",
- "title": "Working with ABFS",
+ "doc": "Getting-Started(v1.2.1)",
+ "title": "Getting-Started(v1.2.1)",
"content": " ",
- "url": "/archives/v1.2.1/docs/getting-started/abfs/#working-with-abfs",
+ "url": "/archives/v1.2.1/docs/getting-started/",
- "relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#working-with-abfs"
+ "relUrl": "/archives/v1.2.1/docs/getting-started/"
},"325": {
- "doc": "Using ABFS with Gluten(v1.2.1)",
- "title": "Configuring ABFS Access Token",
- "content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
- "url": "/archives/v1.2.1/docs/getting-started/abfs/#configuring-abfs-access-token",
+ "doc": "Documentations(v1.2.1)",
+ "title": "Gluten Documents by version",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/#gluten-documents-by-version",
- "relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#configuring-abfs-access-token"
+ "relUrl": "/archives/v1.2.1/docs/#gluten-documents-by-version"
},"326": {
- "doc": "Using ABFS with Gluten(v1.2.1)",
- "title": "Local Caching support",
- "content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
- "url": "/archives/v1.2.1/docs/getting-started/abfs/#local-caching-support",
+ "doc": "Documentations(v1.2.1)",
+ "title": "Documentations(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/",
- "relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#local-caching-support"
+ "relUrl": "/archives/v1.2.1/docs/"
},"327": {
- "doc": "Using ABFS with Gluten(v1.2.1)",
- "title": "Using ABFS with Gluten(v1.2.1)",
- "content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
- "url": "/archives/v1.2.1/docs/getting-started/abfs/",
+ "doc": "Velox Backend(v1.2.1)",
+ "title": "Gluten Documents by version",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/velox-backend/#gluten-documents-by-version",
- "relUrl": "/archives/v1.2.1/docs/getting-started/abfs/"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/#gluten-documents-by-version"
},"328": {
- "doc": "Using ABFS with Gluten",
- "title": "Working with ABFS",
+ "doc": "Velox Backend(v1.2.1)",
+ "title": "Velox Backend(v1.2.1)",
"content": " ",
- "url": "/docs/getting-started/abfs/#working-with-abfs",
+ "url": "/archives/v1.2.1/docs/velox-backend/",
- "relUrl": "/docs/getting-started/abfs/#working-with-abfs"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/"
},"329": {
- "doc": "Using ABFS with Gluten",
- "title": "Configuring ABFS Access Token",
- "content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
- "url": "/docs/getting-started/abfs/#configuring-abfs-access-token",
+ "doc": "Limitations(v1.2.1)",
+ "title": "Velox Backend Limitations",
+ "content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
+ "url": "/archives/v1.2.1/docs/velox-backend/limitations/#velox-backend-limitations",
- "relUrl": "/docs/getting-started/abfs/#configuring-abfs-access-token"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/limitations/#velox-backend-limitations"
},"330": {
- "doc": "Using ABFS with Gluten",
- "title": "Local Caching support",
- "content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
- "url": "/docs/getting-started/abfs/#local-caching-support",
+ "doc": "Limitations(v1.2.1)",
+ "title": "Limitations(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/velox-backend/limitations/",
- "relUrl": "/docs/getting-started/abfs/#local-caching-support"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/limitations/"
},"331": {
- "doc": "Using ABFS with Gluten",
- "title": "Using ABFS with Gluten",
- "content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
- "url": "/docs/getting-started/abfs/",
+ "doc": "Supported Operators & Functions(v1.2.1)",
+ "title": "The Operators and Functions Support Progress",
+ "content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. The total number of fun [...]
+ "url": "/archives/v1.2.1/docs/velox-backend/support/#the-operators-and-functions-support-progress",
- "relUrl": "/docs/getting-started/abfs/"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/support/#the-operators-and-functions-support-progress"
},"332": {
- "doc": "v1.1.1/Build Parameters for Velox Backend",
- "title": "Build Parameters",
- "content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | ON | . | build_protob [...]
- "url": "/archives/v1.1.1/docs/velox/build_parameters/#build-parameters",
+ "doc": "Supported Operators & Functions(v1.2.1)",
+ "title": "Supported Operators & Functions(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/velox-backend/support/",
- "relUrl": "/archives/v1.1.1/docs/velox/build_parameters/#build-parameters"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/support/"
},"333": {
- "doc": "v1.1.1/Build Parameters for Velox Backend",
- "title": "Gluten Jar for Deployment",
- "content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . ",
- "url": "/archives/v1.1.1/docs/velox/build_parameters/#gluten-jar-for-deployment",
+ "doc": "Troubleshooting(v1.2.1)",
+ "title": "Velox Backend Troubleshooting",
+ "content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
+ "url": "/archives/v1.2.1/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting",
- "relUrl": "/archives/v1.1.1/docs/velox/build_parameters/#gluten-jar-for-deployment"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting"
},"334": {
- "doc": "v1.1.1/Build Parameters for Velox Backend",
- "title": "v1.1.1/Build Parameters for Velox Backend",
+ "doc": "Troubleshooting(v1.2.1)",
+ "title": "Troubleshooting(v1.2.1)",
"content": " ",
- "url": "/archives/v1.1.1/docs/velox/build_parameters/",
+ "url": "/archives/v1.2.1/docs/velox-backend/troubleshooting/",
- "relUrl": "/archives/v1.1.1/docs/velox/build_parameters/"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/troubleshooting/"
},"335": {
- "doc": "Using GCS with Gluten(v1.2.1)",
- "title": "Working with GCS",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": " ",
- "url": "/archives/v1.2.1/docs/getting-started/gcs/#working-with-gcs",
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
- "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#working-with-gcs"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
},"336": {
- "doc": "Using GCS with Gluten(v1.2.1)",
- "title": "Installing the gcloud CLI",
- "content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
- "url": "/archives/v1.2.1/docs/getting-started/gcs/#installing-the-gcloud-cli",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Introduction",
+ "content": "Velox backend supports User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/#introduction",
- "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#installing-the-gcloud-cli"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#introduction"
},"337": {
- "doc": "Using GCS with Gluten(v1.2.1)",
- "title": "Configuring GCS using a user account",
- "content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
- "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-user-account",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Create and Build UDF/UDAF library",
+ "content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/#create-and-build-udfudaf-library",
- "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-user-account"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#create-and-build-udfudaf-library"
},"338": {
- "doc": "Using GCS with Gluten(v1.2.1)",
- "title": "Configuring GCS using a credential file",
- "content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
- "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Using UDF/UDAF in Gluten",
+ "content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the library paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/#using-udfudaf-in-gluten",
- "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#using-udfudaf-in-gluten"
},"339": {
- "doc": "Using GCS with Gluten(v1.2.1)",
- "title": "Configuring GCS endpoints",
- "content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
- "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-endpoints",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Try the example",
+ "content": "We provided Velox UDF examples in file MyUDF.cc and UDAF examples in file MyUDAF.cc. You need to build the gluten project with --build_example=ON to get the example libraries./dev/buildbundle-veloxbe.sh --build_examples=ON . Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ . Start spark-shell or spark-sql with below configuration . # Use the `--files` option to upload a library and configure its relative path --files /path/to/glute [...]
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/#try-the-example",
- "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-endpoints"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#try-the-example"
},"340": {
- "doc": "Using GCS with Gluten(v1.2.1)",
- "title": "Configuring GCS max retry count",
- "content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
- "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-count",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Configurations",
+ "content": "| Parameters | Description | . | spark.gluten.sql.columnar.backend.velox.udfLibraryPaths | Path to the udf/udaf libraries. | . | spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths | Path to the udf/udaf libraries on driver node. Only applicable on yarn-client mode. | . | spark.gluten.sql.columnar.backend.velox.udfAllowTypeConversion | Whether to inject possible cast to convert mismatched data types from input to one registered signatures. | . ",
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/#configurations",
- "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-count"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#configurations"
},"341": {
- "doc": "Using GCS with Gluten(v1.2.1)",
- "title": "Configuring GCS max retry time",
- "content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retring (10s, 1m, etc). ",
- "url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-time",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Pandas UDFs (a.k.a. Vectorized UDFs)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs",
- "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-time"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs"
},"342": {
- "doc": "Using GCS with Gluten(v1.2.1)",
- "title": "Using GCS with Gluten(v1.2.1)",
- "content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
- "url": "/archives/v1.2.1/docs/getting-started/gcs/",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Introduction",
+ "content": "Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer doc. ",
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/#introduction-1",
- "relUrl": "/archives/v1.2.1/docs/getting-started/gcs/"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#introduction-1"
},"343": {
- "doc": "v1.1.1/Using GCS with Gluten",
- "title": "Working with GCS",
- "content": " ",
- "url": "/archives/v1.1.1/docs/velox/gcs/#working-with-gcs",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Using Pandas UDFs in Gluten with Velox Backend",
+ "content": "Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: . pip3 install pyspark==$SPARK_VERSION cython pip3 install pandas pyarrow . Gluten provides a config to control enable ColumnarArrowEvalPython or not, with true as defalt. spark.gluten.sql.columnar.arrowUdf . Then take following PySpark code for example: . from pyspark.sql.functions import pandas_udf, PandasUDFType import pyspark.sql.functions as [...]
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend",
- "relUrl": "/archives/v1.1.1/docs/velox/gcs/#working-with-gcs"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend"
},"344": {
- "doc": "v1.1.1/Using GCS with Gluten",
- "title": "Installing the gcloud CLI",
- "content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
- "url": "/archives/v1.1.1/docs/velox/gcs/#installing-the-gcloud-cli",
+ "doc": "Velox UDF(v1.2.1)",
+ "title": "Velox UDF(v1.2.1)",
+ "content": " ",
+ "url": "/archives/v1.2.1/docs/velox-backend/udf/",
- "relUrl": "/archives/v1.1.1/docs/velox/gcs/#installing-the-gcloud-cli"
+ "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/"
},"345": {
- "doc": "v1.1.1/Using GCS with Gluten",
- "title": "Configuring GCS using a user account",
- "content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
- "url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-user-account",
+ "doc": "v1.2.1",
+ "title": "Gluten Documents by version",
+ "content": " ",
+ "url": "/archives/v1.2.1/#gluten-documents-by-version",
- "relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-user-account"
+ "relUrl": "/archives/v1.2.1/#gluten-documents-by-version"
},"346": {
- "doc": "v1.1.1/Using GCS with Gluten",
- "title": "Configuring GCS using a credential file",
- "content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
- "url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-credential-file",
+ "doc": "v1.2.1",
+ "title": "v1.2.1",
+ "content": " ",
+ "url": "/archives/v1.2.1/",
- "relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-credential-file"
+ "relUrl": "/archives/v1.2.1/"
},"347": {
- "doc": "v1.1.1/Using GCS with Gluten",
- "title": "Configuring GCS endpoints",
- "content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
- "url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-endpoints",
+ "doc": "Configuration",
+ "title": "Spark Configurations for Gluten Plugin",
+ "content": "There are many configurations could impact the Gluten Plugin performance and can be fine-tuned in Spark. You can add these configurations into spark-defaults.conf to enable or disable the setting. ",
+ "url": "/archives/v1.3.0/configuration#spark-configurations-for-gluten-plugin",
- "relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-endpoints"
+ "relUrl": "/archives/v1.3.0/configuration#spark-configurations-for-gluten-plugin"
},"348": {
- "doc": "v1.1.1/Using GCS with Gluten",
- "title": "v1.1.1/Using GCS with Gluten",
- "content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
- "url": "/archives/v1.1.1/docs/velox/gcs/",
+ "doc": "Configuration",
+ "title": "Spark parameters",
+ "content": "| Parameters | Description | Recommend Setting | . | spark.driver.extraClassPath | To add Gluten Plugin jar file in Spark Driver | /path/to/jar_file | . | spark.executor.extraClassPath | To add Gluten Plugin jar file in Spark Executor | /path/to/jar_file | . | spark.executor.memory | To set up how much memory to be used for Spark Executor. | | . | spark.memory.offHeap.size | To set up how much memory to be used for Java OffHeap. Please notice Gluten Plugin will leverage [...]
+ "url": "/archives/v1.3.0/configuration#spark-parameters",
- "relUrl": "/archives/v1.1.1/docs/velox/gcs/"
+ "relUrl": "/archives/v1.3.0/configuration#spark-parameters"
},"349": {
- "doc": "Using GCS with Gluten",
- "title": "Working with GCS",
- "content": " ",
- "url": "/docs/getting-started/gcs/#working-with-gcs",
+ "doc": "Configuration",
+ "title": "Velox Parameters",
+ "content": "The following configurations are related to Velox settings. | Parameters | Description | Recommend Setting | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The m [...]
+ "url": "/archives/v1.3.0/configuration#velox-parameters",
- "relUrl": "/docs/getting-started/gcs/#working-with-gcs"
+ "relUrl": "/archives/v1.3.0/configuration#velox-parameters"
},"350": {
- "doc": "Using GCS with Gluten",
- "title": "Installing the gcloud CLI",
- "content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
- "url": "/docs/getting-started/gcs/#installing-the-gcloud-cli",
+ "doc": "Configuration",
+ "title": "Configuration",
+ "content": " ",
+ "url": "/archives/v1.3.0/configuration",
- "relUrl": "/docs/getting-started/gcs/#installing-the-gcloud-cli"
+ "relUrl": "/archives/v1.3.0/configuration"
},"351": {
- "doc": "Using GCS with Gluten",
- "title": "Configuring GCS using a user account",
- "content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
- "url": "/docs/getting-started/gcs/#configuring-gcs-using-a-user-account",
+ "doc": "CPP Code Style",
+ "title": "Gluten CPP Core Guidelines",
+ "content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#gluten-cpp-core-guidelines",
- "relUrl": "/docs/getting-started/gcs/#configuring-gcs-using-a-user-account"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#gluten-cpp-core-guidelines"
},"352": {
- "doc": "Using GCS with Gluten",
- "title": "Configuring GCS using a credential file",
- "content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
- "url": "/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file",
+ "doc": "CPP Code Style",
+ "title": "Philosophy",
+ "content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#philosophy",
- "relUrl": "/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#philosophy"
},"353": {
- "doc": "Using GCS with Gluten",
- "title": "Configuring GCS endpoints",
- "content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
- "url": "/docs/getting-started/gcs/#configuring-gcs-endpoints",
+ "doc": "CPP Code Style",
+ "title": "Code Formatting",
+ "content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#code-formatting",
- "relUrl": "/docs/getting-started/gcs/#configuring-gcs-endpoints"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#code-formatting"
},"354": {
- "doc": "Using GCS with Gluten",
- "title": "Configuring GCS max retry count",
- "content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
- "url": "/docs/getting-started/gcs/#configuring-gcs-max-retry-count",
+ "doc": "CPP Code Style",
+ "title": "Naming Conventions",
+ "content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#naming-conventions",
- "relUrl": "/docs/getting-started/gcs/#configuring-gcs-max-retry-count"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#naming-conventions"
},"355": {
- "doc": "Using GCS with Gluten",
- "title": "Configuring GCS max retry time",
- "content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retring (10s, 1m, etc). ",
- "url": "/docs/getting-started/gcs/#configuring-gcs-max-retry-time",
+ "doc": "CPP Code Style",
+ "title": "Designs",
+ "content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#designs",
- "relUrl": "/docs/getting-started/gcs/#configuring-gcs-max-retry-time"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#designs"
},"356": {
- "doc": "Using GCS with Gluten",
- "title": "Using GCS with Gluten",
- "content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
- "url": "/docs/getting-started/gcs/",
+ "doc": "CPP Code Style",
+ "title": "Source File & Header File",
+ "content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#source-file--header-file",
- "relUrl": "/docs/getting-started/gcs/"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#source-file--header-file"
},"357": {
- "doc": "Velox Local Caching(v1.2.1)",
- "title": "Velox Local Caching(v1.2.1)",
- "content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
- "url": "/archives/v1.2.1/docs/getting-started/localcache/",
+ "doc": "CPP Code Style",
+ "title": "Class",
+ "content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#class",
- "relUrl": "/archives/v1.2.1/docs/getting-started/localcache/"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#class"
},"358": {
- "doc": "Velox Local Caching",
- "title": "Velox Local Caching",
- "content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
- "url": "/docs/getting-started/localcache/",
+ "doc": "CPP Code Style",
+ "title": "Function",
+ "content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#function",
- "relUrl": "/docs/getting-started/localcache/"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#function"
},"359": {
- "doc": "Using S3 with Gluten(v1.2.1)",
- "title": "Working with S3",
- "content": " ",
- "url": "/archives/v1.2.1/docs/getting-started/s3/#working-with-s3",
+ "doc": "CPP Code Style",
+ "title": "Variable",
+ "content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#variable",
- "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#working-with-s3"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#variable"
},"360": {
- "doc": "Using S3 with Gluten(v1.2.1)",
- "title": "Configuring S3 endpoint",
- "content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
- "url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-endpoint",
+ "doc": "CPP Code Style",
+ "title": "Constant",
+ "content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#constant",
- "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-endpoint"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#constant"
},"361": {
- "doc": "Using S3 with Gluten(v1.2.1)",
- "title": "Configuring S3 instance credentials",
- "content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
- "url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-instance-credentials",
+ "doc": "CPP Code Style",
+ "title": "Macro",
+ "content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#macro",
- "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-instance-credentials"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#macro"
},"362": {
- "doc": "Using S3 with Gluten(v1.2.1)",
- "title": "Configuring S3 IAM roles",
- "content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
- "url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-iam-roles",
+ "doc": "CPP Code Style",
+ "title": "Namespace",
+ "content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#namespace",
- "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-iam-roles"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#namespace"
},"363": {
- "doc": "Using S3 with Gluten(v1.2.1)",
- "title": "Other authentatication methods are not supported yet",
- "content": ". | AWS temporary credential | . ",
- "url": "/archives/v1.2.1/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet",
+ "doc": "CPP Code Style",
+ "title": "Resource Management",
+ "content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#resource-management",
- "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#resource-management"
},"364": {
- "doc": "Using S3 with Gluten(v1.2.1)",
- "title": "Log granularity of AWS C++ SDK in velox",
- "content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
- "url": "/archives/v1.2.1/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox",
-
- "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox"
- },"365": {
- "doc": "Using S3 with Gluten(v1.2.1)",
- "title": "Local Caching support",
- "content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
- "url": "/archives/v1.2.1/docs/getting-started/s3/#local-caching-support",
+ "doc": "CPP Code Style",
+ "title": "Exception",
+ "content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#exception",
- "relUrl": "/archives/v1.2.1/docs/getting-started/s3/#local-caching-support"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#exception"
+ },"365": {
+ "doc": "CPP Code Style",
+ "title": "Code Comment",
+ "content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#code-comment",
+
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#code-comment"
},"366": {
- "doc": "Using S3 with Gluten(v1.2.1)",
- "title": "Using S3 with Gluten(v1.2.1)",
- "content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
- "url": "/archives/v1.2.1/docs/getting-started/s3/",
+ "doc": "CPP Code Style",
+ "title": "References",
+ "content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/#references",
- "relUrl": "/archives/v1.2.1/docs/getting-started/s3/"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/#references"
},"367": {
- "doc": "v1.1.1/Using S3 with Gluten",
- "title": "Working with S3",
+ "doc": "CPP Code Style",
+ "title": "CPP Code Style",
"content": " ",
- "url": "/archives/v1.1.1/docs/velox/s3/#working-with-s3",
+ "url": "/archives/v1.3.0/developers/cpp-code-style/",
- "relUrl": "/archives/v1.1.1/docs/velox/s3/#working-with-s3"
+ "relUrl": "/archives/v1.3.0/developers/cpp-code-style/"
},"368": {
- "doc": "v1.1.1/Using S3 with Gluten",
- "title": "Configuring S3 endpoint",
- "content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
- "url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-endpoint",
+ "doc": "How To Use Gluten",
+ "title": "How to understand the key work of Gluten?",
+ "content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
+ "url": "/archives/v1.3.0/developers/how-to-use/#how-to-understand-the-key-work-of-gluten",
- "relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-endpoint"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-understand-the-key-work-of-gluten"
},"369": {
- "doc": "v1.1.1/Using S3 with Gluten",
- "title": "Configuring S3 instance credentials",
- "content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
- "url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-instance-credentials",
+ "doc": "How To Use Gluten",
+ "title": "How to debug in Gluten?",
+ "content": " ",
+ "url": "/archives/v1.3.0/developers/how-to-use/#how-to-debug-in-gluten",
- "relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-instance-credentials"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-debug-in-gluten"
},"370": {
- "doc": "v1.1.1/Using S3 with Gluten",
- "title": "Configuring S3 IAM roles",
- "content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
- "url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-iam-roles",
+ "doc": "How To Use Gluten",
+ "title": "1 How to debug C++",
+ "content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
+ "url": "/archives/v1.3.0/developers/how-to-use/#1-how-to-debug-c",
- "relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-iam-roles"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/#1-how-to-debug-c"
},"371": {
- "doc": "v1.1.1/Using S3 with Gluten",
- "title": "Other authentatication methods are not supported yet",
- "content": ". | AWS temporary credential | . ",
- "url": "/archives/v1.1.1/docs/velox/s3/#other-authentatication-methods-are-not-supported-yet",
+ "doc": "How To Use Gluten",
+ "title": "2 How to debug plan validation process",
+ "content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
+ "url": "/archives/v1.3.0/developers/how-to-use/#2-how-to-debug-plan-validation-process",
- "relUrl": "/archives/v1.1.1/docs/velox/s3/#other-authentatication-methods-are-not-supported-yet"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/#2-how-to-debug-plan-validation-process"
},"372": {
- "doc": "v1.1.1/Using S3 with Gluten",
- "title": "Log granularity of AWS C++ SDK in velox",
- "content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
- "url": "/archives/v1.1.1/docs/velox/s3/#log-granularity-of-aws-c-sdk-in-velox",
+ "doc": "How To Use Gluten",
+ "title": "3 How to debug Java/Scala",
+ "content": "wait to add . ",
+ "url": "/archives/v1.3.0/developers/how-to-use/#3-how-to-debug-javascala",
- "relUrl": "/archives/v1.1.1/docs/velox/s3/#log-granularity-of-aws-c-sdk-in-velox"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/#3-how-to-debug-javascala"
},"373": {
- "doc": "v1.1.1/Using S3 with Gluten",
- "title": "Local Caching support",
- "content": "Velox supports a local cache when reading data from HDFS/S3. The feature is very useful if remote storage is slow, e.g., reading from a public S3 bucket and stronger performance is desired. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage, and the future reading requests on already cached blocks will be serviced from local cache files. To enable the local caching feature, below configurations are required: . spark.g [...]
- "url": "/archives/v1.1.1/docs/velox/s3/#local-caching-support",
+ "doc": "How To Use Gluten",
+ "title": "4 How to debug with core-dump",
+ "content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
+ "url": "/archives/v1.3.0/developers/how-to-use/#4-how-to-debug-with-core-dump",
- "relUrl": "/archives/v1.1.1/docs/velox/s3/#local-caching-support"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/#4-how-to-debug-with-core-dump"
},"374": {
- "doc": "v1.1.1/Using S3 with Gluten",
- "title": "v1.1.1/Using S3 with Gluten",
- "content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
- "url": "/archives/v1.1.1/docs/velox/s3/",
+ "doc": "How To Use Gluten",
+ "title": "How to run TPC-H on Velox backend",
+ "content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox with docker by workflow, you can refer to velox_docker.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, [...]
+ "url": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend",
- "relUrl": "/archives/v1.1.1/docs/velox/s3/"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend"
},"375": {
- "doc": "Using S3 with Gluten",
- "title": "Working with S3",
- "content": " ",
- "url": "/docs/getting-started/s3/#working-with-s3",
+ "doc": "How To Use Gluten",
+ "title": "How to run TPC-DS",
+ "content": "wait to add . ",
+ "url": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-ds",
- "relUrl": "/docs/getting-started/s3/#working-with-s3"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-ds"
},"376": {
- "doc": "Using S3 with Gluten",
- "title": "Configuring S3 endpoint",
- "content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
- "url": "/docs/getting-started/s3/#configuring-s3-endpoint",
+ "doc": "How To Use Gluten",
+ "title": "How to track the memory exhaust problem",
+ "content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
+ "url": "/archives/v1.3.0/developers/how-to-use/#how-to-track-the-memory-exhaust-problem",
- "relUrl": "/docs/getting-started/s3/#configuring-s3-endpoint"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-track-the-memory-exhaust-problem"
},"377": {
- "doc": "Using S3 with Gluten",
- "title": "Configuring S3 instance credentials",
- "content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
- "url": "/docs/getting-started/s3/#configuring-s3-instance-credentials",
+ "doc": "How To Use Gluten",
+ "title": "How To Use Gluten",
+ "content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
+ "url": "/archives/v1.3.0/developers/how-to-use/",
- "relUrl": "/docs/getting-started/s3/#configuring-s3-instance-credentials"
+ "relUrl": "/archives/v1.3.0/developers/how-to-use/"
},"378": {
- "doc": "Using S3 with Gluten",
- "title": "Configuring S3 IAM roles",
- "content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
- "url": "/docs/getting-started/s3/#configuring-s3-iam-roles",
+ "doc": "How To Release",
+ "title": "How to Release",
+ "content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
+ "url": "/docs/developers/how-to-release/#how-to-release",
- "relUrl": "/docs/getting-started/s3/#configuring-s3-iam-roles"
+ "relUrl": "/docs/developers/how-to-release/#how-to-release"
},"379": {
- "doc": "Using S3 with Gluten",
- "title": "Other authentatication methods are not supported yet",
- "content": ". | AWS temporary credential | . ",
- "url": "/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet",
+ "doc": "How To Release",
+ "title": "Release Process",
+ "content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
+ "url": "/docs/developers/how-to-release/#release-process",
- "relUrl": "/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet"
+ "relUrl": "/docs/developers/how-to-release/#release-process"
},"380": {
- "doc": "Using S3 with Gluten",
- "title": "Log granularity of AWS C++ SDK in velox",
- "content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
- "url": "/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox",
+ "doc": "How To Release",
+ "title": "How To Release",
+ "content": " ",
+ "url": "/docs/developers/how-to-release/",
- "relUrl": "/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox"
+ "relUrl": "/docs/developers/how-to-release/"
},"381": {
- "doc": "Using S3 with Gluten",
- "title": "Local Caching support",
- "content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
- "url": "/docs/getting-started/s3/#local-caching-support",
+ "doc": "How To Scan Security issues",
+ "title": "How to scan the security issues",
+ "content": "This section outlines the steps to use tools to scan Apache Gluten (incubating) source code and make sure no vulnerability issues in the code. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
+ "url": "/archives/v1.3.0/developers/how-to-security-scan/#how-to-scan-the-security-issues",
- "relUrl": "/docs/getting-started/s3/#local-caching-support"
+ "relUrl": "/archives/v1.3.0/developers/how-to-security-scan/#how-to-scan-the-security-issues"
},"382": {
- "doc": "Using S3 with Gluten",
- "title": "Using S3 with Gluten",
- "content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
- "url": "/docs/getting-started/s3/",
+ "doc": "How To Scan Security issues",
+ "title": "Scan Security Process",
+ "content": "Before every Apache Gluten (incubating) release, we need to ensure there is no vulnerability issue in the source code. We use Trivy as the tool to scan all the security issues. | Install Trivy, please follow the steps to install Trivy: Trivy Installation . | Configuring Trivy, please follow the guide to configure Trivy for specific operation: Trivy Configuration . | Run Trivy File System Scan with the source code. Below is an example about how we run Trivy scan with Apach [...]
+ "url": "/archives/v1.3.0/developers/how-to-security-scan/#scan-security-process",
- "relUrl": "/docs/getting-started/s3/"
+ "relUrl": "/archives/v1.3.0/developers/how-to-security-scan/#scan-security-process"
},"383": {
- "doc": "About",
- "title": "About",
- "content": "This is the base Jekyll theme. You can find out more info about customizing your Jekyll theme, as well as basic Jekyll usage documentation at jekyllrb.com . You can find the source code for Minima at GitHub: jekyll / minima . You can find the source code for Jekyll at GitHub: jekyll / jekyll . ",
- "url": "/about/",
+ "doc": "How To Scan Security issues",
+ "title": "How To Scan Security issues",
+ "content": " ",
+ "url": "/archives/v1.3.0/developers/how-to-security-scan/",
- "relUrl": "/about/"
+ "relUrl": "/archives/v1.3.0/developers/how-to-security-scan/"
},"384": {
- "doc": "Blog",
- "title": "Latest Posts",
- "content": ". | ",
- "url": "/blog/",
+ "doc": "Micro Benchmarks for Velox Backend",
+ "title": "Generate Micro Benchmarks for Velox Backend",
+ "content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
+ "url": "/archives/v1.3.0/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend",
- "relUrl": "/blog/"
+ "relUrl": "/archives/v1.3.0/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend"
},"385": {
- "doc": "Blog",
- "title": "Gluten 1.2.0 Release",
- "content": "On Sept. 3rd 2024, I am thrilled to share that Apache Gluten (incubating) has celebrated its 1st Apache official release, version 1.2.0, featuring the following highlights. | ",
- "url": "/blog/",
+ "doc": "Micro Benchmarks for Velox Backend",
+ "title": "Try the example",
+ "content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
+ "url": "/archives/v1.3.0/developers/microbenchmarks/#try-the-example",
- "relUrl": "/blog/"
+ "relUrl": "/archives/v1.3.0/developers/microbenchmarks/#try-the-example"
},"386": {
- "doc": "Blog",
- "title": "Welcome to Jekyll!",
- "content": "You’ll find this post in your _posts directory. Go ahead and edit it and re-build the site to see your changes. You can rebuild the site in many different ways, but the most common way is to run jekyll serve, which launches a web server and auto-regenerates your site when a file is updated. | . ",
- "url": "/blog/",
+ "doc": "Micro Benchmarks for Velox Backend",
+ "title": "Generate Substrait plan and input for any query",
+ "content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
+ "url": "/archives/v1.3.0/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query",
- "relUrl": "/blog/"
+ "relUrl": "/archives/v1.3.0/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query"
},"387": {
- "doc": "Blog",
- "title": "Blog",
- "content": " ",
- "url": "/blog/",
+ "doc": "Micro Benchmarks for Velox Backend",
+ "title": "Save ouput to parquet to analyze",
+ "content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
+ "url": "/archives/v1.3.0/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze",
- "relUrl": "/blog/"
+ "relUrl": "/archives/v1.3.0/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze"
},"388": {
- "doc": "Build Parameters for Velox Backend(v1.2.1)",
- "title": "Build Parameters",
- "content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | OFF | . | build_proto [...]
- "url": "/archives/v1.2.1/docs/getting-started/build-guide/#build-parameters",
+ "doc": "Micro Benchmarks for Velox Backend",
+ "title": "Add shuffle write process",
+ "content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
+ "url": "/archives/v1.3.0/developers/microbenchmarks/#add-shuffle-write-process",
- "relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/#build-parameters"
+ "relUrl": "/archives/v1.3.0/developers/microbenchmarks/#add-shuffle-write-process"
},"389": {
- "doc": "Build Parameters for Velox Backend(v1.2.1)",
- "title": "Gluten Jar for Deployment",
- "content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . | 3.5.1 | 3.5 | 2.12 | . ",
- "url": "/archives/v1.2.1/docs/getting-started/build-guide/#gluten-jar-for-deployment",
+ "doc": "Micro Benchmarks for Velox Backend",
+ "title": "Simulate Spark with multiple processes and threads",
+ "content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
+ "url": "/archives/v1.3.0/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads",
- "relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/#gluten-jar-for-deployment"
+ "relUrl": "/archives/v1.3.0/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads"
},"390": {
- "doc": "Build Parameters for Velox Backend(v1.2.1)",
- "title": "Build Parameters for Velox Backend(v1.2.1)",
+ "doc": "Micro Benchmarks for Velox Backend",
+ "title": "Micro Benchmarks for Velox Backend",
"content": " ",
- "url": "/archives/v1.2.1/docs/getting-started/build-guide/",
+ "url": "/archives/v1.3.0/developers/microbenchmarks/",
- "relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/"
+ "relUrl": "/archives/v1.3.0/developers/microbenchmarks/"
},"391": {
- "doc": "Build Parameters for Velox Backend",
- "title": "Build Parameters",
- "content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | OFF | . | build_proto [...]
- "url": "/docs/getting-started/build-guide/#build-parameters",
+ "doc": "New To Gluten",
+ "title": "Environment",
+ "content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
+ "url": "/archives/v1.3.0/developers/new-to/#environment",
- "relUrl": "/docs/getting-started/build-guide/#build-parameters"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#environment"
},"392": {
- "doc": "Build Parameters for Velox Backend",
- "title": "Gluten Jar for Deployment",
- "content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . | 3.5.1 | 3.5 | 2.12 | . ",
- "url": "/docs/getting-started/build-guide/#gluten-jar-for-deployment",
+ "doc": "New To Gluten",
+ "title": "Openjdk8",
+ "content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
+ "url": "/archives/v1.3.0/developers/new-to/#openjdk8",
- "relUrl": "/docs/getting-started/build-guide/#gluten-jar-for-deployment"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#openjdk8"
},"393": {
- "doc": "Build Parameters for Velox Backend",
- "title": "Build Parameters for Velox Backend",
- "content": " ",
- "url": "/docs/getting-started/build-guide/",
+ "doc": "New To Gluten",
+ "title": "Maven 3.6.3 or above",
+ "content": "Maven Dowload Page And then set the environment setting. ",
+ "url": "/archives/v1.3.0/developers/new-to/#maven-363-or-above",
- "relUrl": "/docs/getting-started/build-guide/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#maven-363-or-above"
},"394": {
- "doc": "Contact Us",
- "title": "Contact Us",
- "content": "This plugin is still under active development now, and doesn’t have a stable release. Welcome to evaluate it. If you encounter any issues or have any suggestions, please submit to our issue list. We’d love to hear your feedback . Apache has meticulously set up dedicated mailing lists for every project, underscoring the pivotal role of these platforms in fostering communication within the Apache community. Mailing lists serve as the backbone for numerous aspects of communi [...]
- "url": "/contact/",
+ "doc": "New To Gluten",
+ "title": "GCC 9.4 or above",
+ "content": " ",
+ "url": "/archives/v1.3.0/developers/new-to/#gcc-94-or-above",
- "relUrl": "/contact/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#gcc-94-or-above"
},"395": {
- "doc": "Contact Us",
- "title": "How to subscribe Gluten community",
- "content": "To subscribe to the dev@gluten.apache.org mailing list as an example, please follow these steps: . | Send a blank email to dev-subscribe@gluten.apache.org | Check your inbox for an email titled confirm subscribe to dev@gluten.apache.org If not received within a reasonable time, ensure it’s not blocked by your email provider. If not blocked and no reply is received after a while, repeat step 1 | Reply directly to the confirmation email without altering the subject or addin [...]
- "url": "/contact/#how-to-subscribe-gluten-community",
+ "doc": "New To Gluten",
+ "title": "Compile gluten using debug mode",
+ "content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
+ "url": "/archives/v1.3.0/developers/new-to/#compile-gluten-using-debug-mode",
- "relUrl": "/contact/#how-to-subscribe-gluten-community"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#compile-gluten-using-debug-mode"
},"396": {
- "doc": "Contact Us",
- "title": "Issues and Discussions",
- "content": "We use github to track bugs, feature requests, and answer questions. File an issue for a bug or feature request. ",
- "url": "/contact/#issues-and-discussions",
+ "doc": "New To Gluten",
+ "title": "Java/scala code development with Intellij",
+ "content": " ",
+ "url": "/archives/v1.3.0/developers/new-to/#javascala-code-development-with-intellij",
- "relUrl": "/contact/#issues-and-discussions"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#javascala-code-development-with-intellij"
},"397": {
- "doc": "Contributing to Gluten",
- "title": "How to become a committer",
- "content": "To initiate your contributions to Gluten, understand the contribution process—any individual can submit patches, documentation, and examples to the project. The PMC routinely incorporates fresh committers from the pool of active contributors, assessing their contributions to Gluten. The criteria for new committers encompass: . | The Scope of Contribution: To expand contributions beyond pull requests, individuals can engage in various activities including reviewing other p [...]
- "url": "/contributing/#how-to-become-a-committer",
+ "doc": "New To Gluten",
+ "title": "Linux intellij local debug",
+ "content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
+ "url": "/archives/v1.3.0/developers/new-to/#linux-intellij-local-debug",
- "relUrl": "/contributing/#how-to-become-a-committer"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#linux-intellij-local-debug"
},"398": {
- "doc": "Contributing to Gluten",
- "title": "Contributing to Gluten",
- "content": " ",
- "url": "/contributing/",
+ "doc": "New To Gluten",
+ "title": "Windows/Mac intellij remote debug",
+ "content": "If you have Ultimate intellij, you can try to debug remotely. ",
+ "url": "/archives/v1.3.0/developers/new-to/#windowsmac-intellij-remote-debug",
- "relUrl": "/contributing/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#windowsmac-intellij-remote-debug"
},"399": {
- "doc": "Docker script for CentOS 7(v1.2.1)",
- "title": "Docker script for CentOS 7(v1.2.1)",
- "content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git236 \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ autoconf \\ autoconf-archive \\ automake \\ perl-IPC-Cmd \\ libicu-devel \\ [...]
- "url": "/archives/v1.2.1/docs/developers/docker-centos7/",
+ "doc": "New To Gluten",
+ "title": "Set up gluten project",
+ "content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
+ "url": "/archives/v1.3.0/developers/new-to/#set-up-gluten-project",
- "relUrl": "/archives/v1.2.1/docs/developers/docker-centos7/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#set-up-gluten-project"
},"400": {
- "doc": "v1.1.1/Docker script for CentOS 7",
- "title": "v1.1.1/Docker script for CentOS 7",
- "content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ sudo # gluten need maven version >=3.6.3 wget https://downloads.apache.org/m [...]
- "url": "/archives/v1.1.1/developers/docker_centos7/",
+ "doc": "New To Gluten",
+ "title": "Java/Scala code style",
+ "content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
+ "url": "/archives/v1.3.0/developers/new-to/#javascala-code-style",
- "relUrl": "/archives/v1.1.1/developers/docker_centos7/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#javascala-code-style"
},"401": {
- "doc": "Docker script for CentOS 7",
- "title": "Docker script for CentOS 7",
- "content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git236 \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ autoconf \\ autoconf-archive \\ automake \\ perl-IPC-Cmd \\ libicu-devel \\ [...]
- "url": "/docs/developers/docker-centos7/",
+ "doc": "New To Gluten",
+ "title": "CPP code development with Visual Studio Code",
+ "content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
+ "url": "/archives/v1.3.0/developers/new-to/#cpp-code-development-with-visual-studio-code",
- "relUrl": "/docs/developers/docker-centos7/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#cpp-code-development-with-visual-studio-code"
},"402": {
- "doc": "Docker script for CentOS 8(v1.2.1)",
- "title": "Docker script for CentOS 8(v1.2.1)",
- "content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
- "url": "/archives/v1.2.1/docs/developers/docker-centos8/",
+ "doc": "New To Gluten",
+ "title": "Usage",
+ "content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
+ "url": "/archives/v1.3.0/developers/new-to/#usage",
- "relUrl": "/archives/v1.2.1/docs/developers/docker-centos8/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#usage"
},"403": {
- "doc": "v1.1.1/Docker script for CentOS 8",
- "title": "v1.1.1/Docker script for CentOS 8",
- "content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
- "url": "/archives/v1.1.1/developers/docker_centos8",
+ "doc": "New To Gluten",
+ "title": "Usefule notes",
+ "content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
+ "url": "/archives/v1.3.0/developers/new-to/#usefule-notes",
- "relUrl": "/archives/v1.1.1/developers/docker_centos8"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#usefule-notes"
},"404": {
- "doc": "Docker script for CentOS 8",
- "title": "Docker script for CentOS 8",
- "content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
- "url": "/docs/developers/docker-centos8/",
+ "doc": "New To Gluten",
+ "title": "Debug cpp code with coredump",
+ "content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
+ "url": "/archives/v1.3.0/developers/new-to/#debug-cpp-code-with-coredump",
- "relUrl": "/docs/developers/docker-centos8/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#debug-cpp-code-with-coredump"
},"405": {
- "doc": "Docker script for Ubuntu 22.04/20.04(v1.2.1)",
- "title": "Docker script for Ubuntu 22.04/20.04(v1.2.1)",
- "content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
- "url": "/archives/v1.2.1/docs/developers/docker-ubuntu/",
+ "doc": "New To Gluten",
+ "title": "Debug cpp with gdb",
+ "content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
+ "url": "/archives/v1.3.0/developers/new-to/#debug-cpp-with-gdb",
- "relUrl": "/archives/v1.2.1/docs/developers/docker-ubuntu/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#debug-cpp-with-gdb"
},"406": {
- "doc": "v1.1.1/Docker script for Ubuntu 22.04/20.04",
- "title": "v1.1.1/Docker script for Ubuntu 22.04/20.04",
- "content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
- "url": "/archives/v1.1.1/developers/docker_ubuntu/",
+ "doc": "New To Gluten",
+ "title": "Run TPC-H and TPC-DS",
+ "content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_docker.yml . ",
+ "url": "/archives/v1.3.0/developers/new-to/#run-tpc-h-and-tpc-ds",
- "relUrl": "/archives/v1.1.1/developers/docker_ubuntu/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#run-tpc-h-and-tpc-ds"
},"407": {
- "doc": "Docker script for Ubuntu 22.04/20.04",
- "title": "Docker script for Ubuntu 22.04/20.04",
- "content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
- "url": "/docs/developers/docker-ubuntu/",
+ "doc": "New To Gluten",
+ "title": "Run gluten+velox on clean machine",
+ "content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
+ "url": "/archives/v1.3.0/developers/new-to/#run-glutenvelox-on-clean-machine",
- "relUrl": "/docs/developers/docker-ubuntu/"
+ "relUrl": "/archives/v1.3.0/developers/new-to/#run-glutenvelox-on-clean-machine"
},"408": {
- "doc": "Downloads",
- "title": "Download Apache Gluten (incubating)",
- "content": "Gluten is a plugin for Apache Spark to double SparkSQL’s performance. ",
- "url": "/downloads/#download-apache-gluten-incubating",
+ "doc": "New To Gluten",
+ "title": "New To Gluten",
+ "content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
+ "url": "/archives/v1.3.0/developers/new-to/",
- "relUrl": "/downloads/#download-apache-gluten-incubating"
+ "relUrl": "/archives/v1.3.0/developers/new-to/"
},"409": {
- "doc": "Downloads",
- "title": "Instructions",
- "content": "Please select a version of Gluten to download from the list below; we recommend opting for the latest release. Please ensure the integrity of the release by verifying the corresponding SHA-512 hashes, signatures, and the project release KEYS. Instructions for verifying the SHA-512 hashes and signatures can be found in the provided guidelines. ",
- "url": "/downloads/#instructions",
+ "doc": "Substrait Modifications",
+ "title": "Substrait Modifications in Gluten",
+ "content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
+ "url": "/archives/v1.3.0/developers/substrait/#substrait-modifications-in-gluten",
- "relUrl": "/downloads/#instructions"
+ "relUrl": "/archives/v1.3.0/developers/substrait/#substrait-modifications-in-gluten"
},"410": {
- "doc": "Downloads",
- "title": "The Latest Apache Official Release",
- "content": "Please be aware that the binary file was compiled on CentOS7 using an Intel(r) Xeon(r) Gold 6252 processor with static linking. While it may operate on most operating systems and Intel SKYLAKE or later platforms, performance is not guaranteed. For the best performance and platform support, we recommend compiling from source code yourself. | Version | Date | Source | Binary | Release Notes | . | 1.3.0 | Jan. 24 2025 | source(sha512,signature) | spark32-binary(sha512,signat [...]
- "url": "/downloads/#the-latest-apache-official-release",
+ "doc": "Substrait Modifications",
+ "title": "Modifications to algebra.proto",
+ "content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
+ "url": "/archives/v1.3.0/developers/substrait/#modifications-to-algebraproto",
- "relUrl": "/downloads/#the-latest-apache-official-release"
+ "relUrl": "/archives/v1.3.0/developers/substrait/#modifications-to-algebraproto"
},"411": {
- "doc": "Downloads",
- "title": "All Archived Releases",
- "content": "| Version | Date | Source | Binary | Release Notes | . | 1.1.1(non-Apache release) | Mar. 2 2024 | source | binary location | release notes | . | 1.1.0(non-Apache release) | Nov. 30 2023 | source | binary location | release notes | . | 1.0.0(non-Apache release) | Jul. 13 2023 | source | binary location | release notes | . | 0.5.0(non-Apache release) | Apr. 7 2023 | source | binary location | release notes | . ",
- "url": "/downloads/#all-archived-releases",
+ "doc": "Substrait Modifications",
+ "title": "Modifications to type.proto",
+ "content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
+ "url": "/archives/v1.3.0/developers/substrait/#modifications-to-typeproto",
- "relUrl": "/downloads/#all-archived-releases"
+ "relUrl": "/archives/v1.3.0/developers/substrait/#modifications-to-typeproto"
},"412": {
- "doc": "Downloads",
- "title": "License",
- "content": "The software licensed under Apache License 2.0. ",
- "url": "/downloads/#license",
+ "doc": "Substrait Modifications",
+ "title": "Substrait Modifications",
+ "content": " ",
+ "url": "/archives/v1.3.0/developers/substrait/",
- "relUrl": "/downloads/#license"
+ "relUrl": "/archives/v1.3.0/developers/substrait/"
},"413": {
- "doc": "Downloads",
- "title": "Downloads",
- "content": " ",
- "url": "/downloads/",
+ "doc": "Docker script for CentOS 7",
+ "title": "Docker script for CentOS 7",
+ "content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git236 \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ autoconf \\ autoconf-archive \\ automake \\ perl-IPC-Cmd \\ libicu-devel \\ [...]
+ "url": "/archives/v1.3.0/developers/docker-centos7/",
- "relUrl": "/downloads/"
+ "relUrl": "/archives/v1.3.0/developers/docker-centos7/"
},"414": {
- "doc": "Developers(v1.2.1)",
- "title": "Gluten Developers",
- "content": "This document provides a developer overview of the project and covers the following topics: . ",
- "url": "/archives/v1.2.1/docs/developers/#gluten-developers",
+ "doc": "Docker script for CentOS 8",
+ "title": "Docker script for CentOS 8",
+ "content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
+ "url": "/archives/v1.3.0/developers/docker-centos8/",
- "relUrl": "/archives/v1.2.1/docs/developers/#gluten-developers"
+ "relUrl": "/archives/v1.3.0/developers/docker-centos8/"
},"415": {
- "doc": "Developers(v1.2.1)",
- "title": "Developers(v1.2.1)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/developers/",
+ "doc": "Docker script for Ubuntu 22.04/20.04",
+ "title": "Docker script for Ubuntu 22.04/20.04",
+ "content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
+ "url": "/archives/v1.3.0/developers/docker-ubuntu/",
- "relUrl": "/archives/v1.2.1/docs/developers/"
+ "relUrl": "/archives/v1.3.0/developers/docker-ubuntu/"
},"416": {
- "doc": "Getting-Started(v1.2.1)",
- "title": "Getting Started with Gluten for Apache Spark",
- "content": " ",
- "url": "/archives/v1.2.1/docs/getting-started/#getting-started-with-gluten-for-apache-spark",
+ "doc": "Developers",
+ "title": "Gluten Developers",
+ "content": "This document provides a developer overview of the project and covers the following topics: . ",
+ "url": "/archives/v1.3.0/developers#gluten-developers",
- "relUrl": "/archives/v1.2.1/docs/getting-started/#getting-started-with-gluten-for-apache-spark"
+ "relUrl": "/archives/v1.3.0/developers#gluten-developers"
},"417": {
- "doc": "Getting-Started(v1.2.1)",
- "title": "Getting-Started(v1.2.1)",
+ "doc": "Developers",
+ "title": "Developers",
"content": " ",
- "url": "/archives/v1.2.1/docs/getting-started/",
+ "url": "/archives/v1.3.0/developers",
- "relUrl": "/archives/v1.2.1/docs/getting-started/"
+ "relUrl": "/archives/v1.3.0/developers"
},"418": {
- "doc": "Velox Backend(v1.2.1)",
- "title": "Gluten Documents by version",
- "content": " ",
- "url": "/archives/v1.2.1/docs/velox-backend/#gluten-documents-by-version",
+ "doc": "Velox Function Development",
+ "title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
+ "content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
+ "url": "/archives/v1.3.0/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/#gluten-documents-by-version"
+ "relUrl": "/archives/v1.3.0/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
},"419": {
- "doc": "Velox Backend(v1.2.1)",
- "title": "Velox Backend(v1.2.1)",
+ "doc": "Velox Function Development",
+ "title": "Velox Function Development",
"content": " ",
- "url": "/archives/v1.2.1/docs/velox-backend/",
+ "url": "/archives/v1.3.0/developers/velox-function-dev/",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/"
+ "relUrl": "/archives/v1.3.0/developers/velox-function-dev/"
},"420": {
- "doc": "Documentations(v1.2.1)",
- "title": "Gluten Documents by version",
- "content": " ",
- "url": "/archives/v1.2.1/docs/#gluten-documents-by-version",
+ "doc": "Getting start with ClickHouse Backend",
+ "title": "ClickHouse Backend",
+ "content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
+ "url": "/archives/v1.3.0/getting-started/clickhouse-backend/#clickhouse-backend",
- "relUrl": "/archives/v1.2.1/docs/#gluten-documents-by-version"
+ "relUrl": "/archives/v1.3.0/getting-started/clickhouse-backend/#clickhouse-backend"
},"421": {
- "doc": "Documentations(v1.2.1)",
- "title": "Documentations(v1.2.1)",
+ "doc": "Getting start with ClickHouse Backend",
+ "title": "Getting start with ClickHouse Backend",
"content": " ",
- "url": "/archives/v1.2.1/docs/",
+ "url": "/archives/v1.3.0/getting-started/clickhouse-backend/",
- "relUrl": "/archives/v1.2.1/docs/"
+ "relUrl": "/archives/v1.3.0/getting-started/clickhouse-backend/"
},"422": {
- "doc": "v1.2.1",
- "title": "Gluten Documents by version",
- "content": " ",
- "url": "/archives/v1.2.1/#gluten-documents-by-version",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Supported Version",
+ "content": "| Type | Version | . | Spark | 3.2.2, 3.3.1, 3.4.2, 3.5.1 | . | OS | Ubuntu20.04/22.04, Centos7/8 | . | jdk | openjdk8/jdk17 | . | scala | 2.12 | . ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#supported-version",
- "relUrl": "/archives/v1.2.1/#gluten-documents-by-version"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#supported-version"
},"423": {
- "doc": "v1.2.1",
- "title": "v1.2.1",
- "content": " ",
- "url": "/archives/v1.2.1/",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Prerequisite",
+ "content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos7/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.2 and 3.5.1. We need to set up the JAVA_HOME env. Currently, Gluten supports java 8 and java 17. For x86_64 . ## make sure jdk8 is used export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#prerequisite",
- "relUrl": "/archives/v1.2.1/"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#prerequisite"
},"424": {
- "doc": "v1.1.1/Developers",
- "title": "Gluten Developers",
- "content": "This document provides a developer overview of the project and covers the following topics: . ",
- "url": "/archives/v1.1.1/developers/#gluten-developers",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Build Gluten with Velox Backend",
+ "content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . First time build for all supported spark versions./dev/buildbundle-veloxbe.sh . After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and setting up build dependencies./dev/buildbundle-veloxbe.sh --enable_ep [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-velox-backend",
- "relUrl": "/archives/v1.1.1/developers/#gluten-developers"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-velox-backend"
},"425": {
- "doc": "v1.1.1/Developers",
- "title": "v1.1.1/Developers",
- "content": " ",
- "url": "/archives/v1.1.1/developers/",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Dependency library deployment",
+ "content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#dependency-library-deployment",
- "relUrl": "/archives/v1.1.1/developers/"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#dependency-library-deployment"
},"426": {
- "doc": "v1.1.1/Documentation",
- "title": "Gluten Documents by version",
- "content": " ",
- "url": "/archives/v1.1.1/docs/#gluten-documents-by-version",
+ "doc": "Getting Start with Velox Backend",
+ "title": "HDFS support",
+ "content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#hdfs-support",
- "relUrl": "/archives/v1.1.1/docs/#gluten-documents-by-version"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#hdfs-support"
},"427": {
- "doc": "v1.1.1/Documentation",
- "title": "v1.1.1/Documentation",
- "content": " ",
- "url": "/archives/v1.1.1/docs/",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Azure Blob File System (ABFS) support",
+ "content": "Velox supports ABFS with the open source Azure SDK for C++ and Gluten uses the Velox ABFS connector to connect with ABFS. The build option for ABFS (enable_abfs) must be set to enable this feature as listed below. cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_abfs=ON . Please refer Velox ABFS part for more detailed configurations. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#azure-blob-file-system-abfs-support",
- "relUrl": "/archives/v1.1.1/docs/"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#azure-blob-file-system-abfs-support"
},"428": {
- "doc": "v1.1.1",
- "title": "Gluten Documents by version",
- "content": " ",
- "url": "/archives/v1.1.1/#gluten-documents-by-version",
+ "doc": "Getting Start with Velox Backend",
+ "title": "AWS S3 support",
+ "content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#aws-s3-support",
- "relUrl": "/archives/v1.1.1/#gluten-documents-by-version"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#aws-s3-support"
},"429": {
- "doc": "v1.1.1",
- "title": "v1.1.1",
- "content": " ",
- "url": "/archives/v1.1.1/",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Celeborn support",
+ "content": "Gluten with velox backend supports Celeborn as remote shuffle service. Currently, the supported Celeborn versions are 0.3.x, 0.4.x and 0.5.0. Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable celeborn profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests . Then add the Gluten and Spa [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#celeborn-support",
- "relUrl": "/archives/v1.1.1/"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#celeborn-support"
},"430": {
- "doc": "Archives",
- "title": "Gluten Documents by version",
- "content": " ",
- "url": "/archives/#gluten-documents-by-version",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Uniffle support",
+ "content": "Uniffle with velox backend supports Uniffle as remote shuffle service. Currently, the supported Uniffle versions are 0.8.0. First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with uniffle. When compiling the Gluten Java module, it’s required to enable uniffle profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests . Then add the Uniffle and Spark Celeborn Client packages to your Spark application’s classpath(usu [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#uniffle-support",
- "relUrl": "/archives/#gluten-documents-by-version"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#uniffle-support"
},"431": {
- "doc": "Archives",
- "title": "Archives",
- "content": " ",
- "url": "/archives/",
+ "doc": "Getting Start with Velox Backend",
+ "title": "DeltaLake Support",
+ "content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta-XX-SNAPSHOT.jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with vel [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#deltalake-support",
- "relUrl": "/archives/"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#deltalake-support"
},"432": {
- "doc": "Developers",
- "title": "Gluten Developers",
- "content": "This document provides a developer overview of the project and covers the following topics: . ",
- "url": "/docs/developers/#gluten-developers",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Iceberg Support",
+ "content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan’s fallback. After the two steps, you can query [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#iceberg-support",
- "relUrl": "/docs/developers/#gluten-developers"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#iceberg-support"
},"433": {
- "doc": "Developers",
- "title": "Developers",
- "content": " ",
- "url": "/docs/developers/",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Coverage",
+ "content": "Spark3.3 has 387 functions in total. ~240 are commonly used. To get the support status of all Spark built-in functions, please refer to Velox Backend’s Supported Operators & Functions. Velox doesn’t support ANSI mode), so as Gluten. Once ANSI mode is enabled in Spark config, Gluten will fallback to Vanilla Spark. To identify what can be offloaded in a query and detailed fallback reasons, user can follow below steps to retrieve corresponding logs. 1) Enable Gluten by p [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#coverage",
- "relUrl": "/docs/developers/"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#coverage"
},"434": {
- "doc": "Getting-Started",
- "title": "Getting Started with Gluten for Apache Spark",
- "content": " ",
- "url": "/docs/getting-started/#getting-started-with-gluten-for-apache-spark",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Spill (Experimental)",
+ "content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#spill-experimental",
- "relUrl": "/docs/getting-started/#getting-started-with-gluten-for-apache-spark"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#spill-experimental"
},"435": {
- "doc": "Getting-Started",
- "title": "Getting-Started",
- "content": " ",
- "url": "/docs/getting-started/",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
+ "content": "Please check the VeloxNativeUDF.md for more detailed usage and configurations. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
- "relUrl": "/docs/getting-started/"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
},"436": {
- "doc": "Velox Backend",
- "title": "Gluten Documents by version",
- "content": " ",
- "url": "/docs/velox-backend/#gluten-documents-by-version",
+ "doc": "Getting Start with Velox Backend",
+ "title": "High-Bandwidth Memory (HBM) support",
+ "content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#high-bandwidth-memory-hbm-support",
- "relUrl": "/docs/velox-backend/#gluten-documents-by-version"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#high-bandwidth-memory-hbm-support"
},"437": {
- "doc": "Velox Backend",
- "title": "Velox Backend",
- "content": " ",
- "url": "/docs/velox-backend/",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Build Gluten with HBM",
+ "content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.2 and 3.5.1./dev/buildbundle-veloxbe.sh -- [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-hbm",
- "relUrl": "/docs/velox-backend/"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-hbm"
},"438": {
- "doc": "Documentations",
- "title": "Gluten Documents by version",
- "content": " ",
- "url": "/docs/#gluten-documents-by-version",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Configure and enable HBM in Spark Application",
+ "content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application",
- "relUrl": "/docs/#gluten-documents-by-version"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application"
},"439": {
- "doc": "Documentations",
- "title": "Documentations",
- "content": " ",
- "url": "/docs/",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Intel® QuickAssist Technology (QAT) support",
+ "content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#intel-quickassist-technology-qat-support",
- "relUrl": "/docs/"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#intel-quickassist-technology-qat-support"
},"440": {
- "doc": "Home",
- "title": "Overview",
- "content": "Apache Gluten(incubating) is a middle layer responsible for offloading JVM-based SQL engines’ execution to native engines. ",
- "url": "/#overview",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Software Requirements",
+ "content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#software-requirements",
- "relUrl": "/#overview"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#software-requirements"
},"441": {
- "doc": "Home",
- "title": "1 Introduction",
- "content": " ",
- "url": "/#1-introduction",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Build Gluten with QAT",
+ "content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-qat",
- "relUrl": "/#1-introduction"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-qat"
},"442": {
- "doc": "Home",
- "title": "1.1 Problem Statement",
- "content": "Apache Spark is a stable, mature project that has been developed for many years. It is one of the best frameworks to scale out for processing petabyte-scale datasets. However, the Spark community has had to address performance challenges that require various optimizations over time. As a key optimization in Spark 2.0, Whole Stage Code Generation is introduced to replace Volcano Model, which achieves 2x speedup. Henceforth, most optimizations are at query plan level. Singl [...]
- "url": "/#11-problem-statement",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Enable QAT with Gzip/Zstd for shuffle compression",
+ "content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression",
- "relUrl": "/#11-problem-statement"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression"
},"443": {
- "doc": "Home",
- "title": "1.2 Gluten’s Solution",
- "content": "“Gluten” is Latin for glue. The main goal of Gluten project is to “glue” native libraries with SparkSQL. Thus, we can benefit from high scalability of Spark SQL framework and high performance of native libraries. The basic rule of Gluten’s design is that we would reuse spark’s whole control flow and as many JVM code as possible but offload the compute-intensive data processing part to native code. Here is what Gluten does: . | Transform Spark’s whole stage physical plan t [...]
- "url": "/#12-glutens-solution",
+ "doc": "Getting Start with Velox Backend",
+ "title": "QAT driver references",
+ "content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#qat-driver-references",
- "relUrl": "/#12-glutens-solution"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#qat-driver-references"
},"444": {
- "doc": "Home",
- "title": "1.3 Target User",
- "content": "Gluten’s target user is anyone who wants to accelerate SparkSQL fundamentally. As a plugin to Spark, Gluten doesn’t require any change for dataframe API or SQL query, but only requires user to make correct configuration. See Gluten configuration properties here. ",
- "url": "/#13-target-user",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
+ "content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support",
- "relUrl": "/#13-target-user"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support"
},"445": {
- "doc": "Home",
- "title": "1.4 References",
- "content": "You can click below links for more related information. | Gluten References | . ",
- "url": "/#14-references",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Build Gluten with IAA",
+ "content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-iaa",
- "relUrl": "/#14-references"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-iaa"
},"446": {
- "doc": "Home",
- "title": "2 Architecture",
- "content": "The overview chart is like below. Substrait provides a well-defined cross-language specification for data compute operations (see more details here). Spark physical plan is transformed to Substrait plan. Then Substrait plan is passed to native through JNI call. On native side, the native operator chain will be built out and offloaded to native engine. Gluten will return Columnar Batch to Spark and Spark Columnar API (since Spark-3.0) will be used at execution time. Gluten [...]
- "url": "/#2-architecture",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Enable IAA with Gzip Compression for shuffle compression",
+ "content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression",
- "relUrl": "/#2-architecture"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression"
},"447": {
- "doc": "Home",
- "title": "3 How to Use",
- "content": "There are two methods for utilizing Gluten. The first is to use a pre-built JAR for a quick test of Gluten’s acceleration capabilities. The second is to compile the JAR yourself, ensuring it runs on your target platform and delivers the best possible performance. ",
- "url": "/#3-how-to-use",
+ "doc": "Getting Start with Velox Backend",
+ "title": "IAA references",
+ "content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#iaa-references",
- "relUrl": "/#3-how-to-use"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#iaa-references"
},"448": {
- "doc": "Home",
- "title": "3.1 Use a pre-Built Jar",
- "content": "One way is to use released jar. Here is a simple example. Currently, only centos7/8 and ubuntu20.04/22.04 are well supported. You can find a pre-built gluten release jar in Apache Gluten Download. Please be aware that the pre-built JAR is compiled using static linking via vcpkg, based on the Intel® Xeon® Gold 6252 processor. It should be compatible with most operating systems, including CentOS and Ubuntu, although performance is not guaranteed. For the optimal performance [...]
- "url": "/#31-use-a-pre-built-jar",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
+ "content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
- "relUrl": "/#31-use-a-pre-built-jar"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
},"449": {
- "doc": "Home",
- "title": "3.2 Custom Build",
- "content": "To utilize Gluten with Spark, you can compile Gluten from the source and then configure it to enable the Gluten plugin. Below is a straightforward example. For more comprehensive instructions, please refer to the detailed guidance provided in the corresponding backend section. export gluten_jar = /PATH/TO/GLUTEN/backends-velox/target/<gluten-jar> spark-shell \\ --master yarn --deploy-mode client \\ --conf spark.plugins=org.apache.gluten.GlutenPlugin \\ --conf spark. [...]
- "url": "/#32-custom-build",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Data preparation",
+ "content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#data-preparation",
- "relUrl": "/#32-custom-build"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#data-preparation"
},"450": {
- "doc": "Home",
- "title": "3.2.1 Build and install Gluten with Velox backend",
- "content": "If you want to use Gluten Velox backend, see Build with Velox to build and install the necessary libraries. You can also find more information in Gluten with Velox backend and velox. ",
- "url": "/#321-build-and-install-gluten-with-velox-backend",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Submit the Spark SQL job",
+ "content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#submit-the-spark-sql-job",
- "relUrl": "/#321-build-and-install-gluten-with-velox-backend"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#submit-the-spark-sql-job"
},"451": {
- "doc": "Home",
- "title": "3.2.2 Build and install Gluten with ClickHouse backend",
- "content": "If you want to use Gluten ClickHouse backend, see Build with ClickHouse Backend. ClickHouse backend is developed by Kyligence, please visit Kyligence’s ClickHouse for more infomation. ",
- "url": "/#322-build-and-install-gluten-with-clickhouse-backend",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Result",
+ "content": "wholestagetransformer indicates that the offloading works. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#result",
- "relUrl": "/#322-build-and-install-gluten-with-clickhouse-backend"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#result"
},"452": {
- "doc": "Home",
- "title": "3.2.3 Build options",
- "content": "See Build Parameters. ",
- "url": "/#323-build-options",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Performance",
+ "content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#performance",
- "relUrl": "/#323-build-options"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#performance"
},"453": {
- "doc": "Home",
- "title": "4 Join the Community",
- "content": " ",
- "url": "/#4-join-the-community",
+ "doc": "Getting Start with Velox Backend",
+ "title": "External reference setup",
+ "content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#external-reference-setup",
- "relUrl": "/#4-join-the-community"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#external-reference-setup"
},"454": {
- "doc": "Home",
- "title": "Contact Us",
- "content": "Gluten was initiated by Intel and Kyligence in 2022. Several companies are also actively participating in the development, such as BIGO, Meituan, Alibaba Cloud, NetEase, Baidu, Microsoft, etc. If you are interested in Gluten project, please contact and subscribe below mailing lists for further discussion. Please see contact us for more information. ",
- "url": "/#contact-us",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Gluten UI",
+ "content": " ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#gluten-ui",
- "relUrl": "/#contact-us"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#gluten-ui"
},"455": {
- "doc": "Home",
- "title": "Source Code",
- "content": "Please see gluten source code for more information. ",
- "url": "/#source-code",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Gluten event",
+ "content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#gluten-event",
- "relUrl": "/#source-code"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#gluten-event"
},"456": {
- "doc": "Home",
- "title": "How to Contribute to Gluten",
- "content": "Please see contributing guide about how to make contributions. ",
- "url": "/#how-to-contribute-to-gluten",
+ "doc": "Getting Start with Velox Backend",
+ "title": "SQL tab",
+ "content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#sql-tab",
- "relUrl": "/#how-to-contribute-to-gluten"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#sql-tab"
},"457": {
- "doc": "Home",
- "title": "Wechat Group",
- "content": "There is a Wechat group (in Chinese) which may be more friendly for PRC developers/users. Due to the limitation of wechat group, please mail to dev@gluten.apache.org. ",
- "url": "/#wechat-group",
+ "doc": "Getting Start with Velox Backend",
+ "title": "History server",
+ "content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#history-server",
- "relUrl": "/#wechat-group"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#history-server"
},"458": {
- "doc": "Home",
- "title": "Slack channel",
- "content": "For Velox backend, we recommend to use velox community. ",
- "url": "/#slack-channel",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Native plan string",
+ "content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-string",
- "relUrl": "/#slack-channel"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-string"
},"459": {
- "doc": "Home",
- "title": "5 Performance",
- "content": "We use TPCH-like and TPCDS-like as decison support benchmarks to evaluate Gluten’s performance. TPCH-like is a query set for modified from TPC-H benchmark and TPCDS-like is a query set for modified from TPC-DS benchmark. We use Parquet file format for Velox testing & MergeTree file format for Clickhouse testing, compared to Parquet file format as baseline. See Decision Support Benchmark. ",
- "url": "/#5-performance",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Native plan with stats",
+ "content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-with-stats",
- "relUrl": "/#5-performance"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-with-stats"
},"460": {
- "doc": "Home",
- "title": "Gluten + Velox backend Performance",
- "content": "The below test environment: single node with 3TB data on Intel® Xeon® Platinum 8592+; Spark-3.3.1 for both baseline and Gluten. The TPCH-like result (tested in Mar. 2024) shows an overall speedup of 3.34x and up to 23.45x speedup in a single query with Gluten + Velox backend used. The TPCDS-like result (tested in Mar. 2024) shows an overall speedup of 3.02x and up to 13.75x speedup in a single query with Gluten + Velox backend used. Notices & Disclaimers . Performance [...]
- "url": "/#gluten--velox-backend-performance",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Gluten Implicits",
+ "content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/#gluten-implicits",
- "relUrl": "/#gluten--velox-backend-performance"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/#gluten-implicits"
},"461": {
- "doc": "Home",
- "title": "Gluten + ClickHouse backend Performance",
- "content": "The below testing environment: a 8-nodes AWS cluster with 1TB data; Spark-3.1.1 for both baseline and Gluten. The Decision Support Benchmark1 result shows an average speedup of 2.12x and up to 3.48x speedup with Gluten Clickhouse backend. ",
- "url": "/#gluten--clickhouse-backend-performance",
+ "doc": "Getting Start with Velox Backend",
+ "title": "Getting Start with Velox Backend",
+ "content": " ",
+ "url": "/archives/v1.3.0/getting-started/velox-backend/",
- "relUrl": "/#gluten--clickhouse-backend-performance"
+ "relUrl": "/archives/v1.3.0/getting-started/velox-backend/"
},"462": {
- "doc": "Home",
- "title": "Thanks to our contributors",
- "content": ". ",
- "url": "/#thanks-to-our-contributors",
+ "doc": "Using ABFS with Gluten",
+ "title": "Working with ABFS",
+ "content": " ",
+ "url": "/archives/v1.3.0/getting-started/abfs/#working-with-abfs",
- "relUrl": "/#thanks-to-our-contributors"
+ "relUrl": "/archives/v1.3.0/getting-started/abfs/#working-with-abfs"
},"463": {
- "doc": "Home",
- "title": "Home",
- "content": " ",
- "url": "/",
+ "doc": "Using ABFS with Gluten",
+ "title": "Configuring ABFS Access Token",
+ "content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
+ "url": "/archives/v1.3.0/getting-started/abfs/#configuring-abfs-access-token",
- "relUrl": "/"
+ "relUrl": "/archives/v1.3.0/getting-started/abfs/#configuring-abfs-access-token"
},"464": {
- "doc": "Powered by Gluten",
- "title": "Project and Product Name using “Gluten”",
- "content": "Organizations developing products and projects for use with Apache Gluten (incubating), as well as related marketing materials, should ensure they respect the trademark of “Apache Gluten” and its logo. For comprehensive and authoritative guidance on the proper usage of ASF trademarks, please refer to the ASF Trademarks Guidance and the associated FAQ. ",
- "url": "/poweredby/#project-and-product-name-using-gluten",
+ "doc": "Using ABFS with Gluten",
+ "title": "Local Caching support",
+ "content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
+ "url": "/archives/v1.3.0/getting-started/abfs/#local-caching-support",
- "relUrl": "/poweredby/#project-and-product-name-using-gluten"
+ "relUrl": "/archives/v1.3.0/getting-started/abfs/#local-caching-support"
},"465": {
- "doc": "Powered by Gluten",
- "title": "Companies and Organizations",
- "content": "To add yourself to the list, please email dev@gluten.apache.org with the following information: your organization name, URL, a list of the Gluten components you are using, and a brief description of your use case. Sorted by Company Name . | Company Name | Description | Link | . | Alibaba | Alibaba has chosen Gluten as a part of their public cloud solution. | | . | Baidu | Baidu has used gluten in multiple business scenarios. | | . | Bytedance | EMR in Bytedance’s Volc [...]
- "url": "/poweredby/#companies-and-organizations",
+ "doc": "Using ABFS with Gluten",
+ "title": "Using ABFS with Gluten",
+ "content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
+ "url": "/archives/v1.3.0/getting-started/abfs/",
- "relUrl": "/poweredby/#companies-and-organizations"
+ "relUrl": "/archives/v1.3.0/getting-started/abfs/"
},"466": {
- "doc": "Powered by Gluten",
- "title": "Powered by Gluten",
+ "doc": "Using GCS with Gluten",
+ "title": "Working with GCS",
"content": " ",
- "url": "/poweredby/",
+ "url": "/archives/v1.3.0/getting-started/gcs/#working-with-gcs",
- "relUrl": "/poweredby/"
+ "relUrl": "/archives/v1.3.0/getting-started/gcs/#working-with-gcs"
},"467": {
- "doc": "Gluten References",
- "title": "Gluten related Use Cases and Publications",
- "content": "For more information on Apache Gluten (incubating), including related articles and videos, please refer to the following links. | Title | Company | Publish Date | Reference | . | Gluten: A middle layer to offload Spark SQL to Native | Intel | Jul. 2022 | Link | . | Best Exploration of Columnar Shuffle Design | Intel | Jul. 2022 | Link | . | Gluten Introduction | Intel | Sept. 2022 | Link | . | Introducing Velox | Meta | Mar. 2023 | Link | . | Gluten in Kyligence(in Chines [...]
- "url": "/references/#gluten-related-use-cases-and-publications",
+ "doc": "Using GCS with Gluten",
+ "title": "Installing the gcloud CLI",
+ "content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
+ "url": "/archives/v1.3.0/getting-started/gcs/#installing-the-gcloud-cli",
- "relUrl": "/references/#gluten-related-use-cases-and-publications"
+ "relUrl": "/archives/v1.3.0/getting-started/gcs/#installing-the-gcloud-cli"
},"468": {
- "doc": "Gluten References",
- "title": "Gluten References",
- "content": " ",
- "url": "/references/",
+ "doc": "Using GCS with Gluten",
+ "title": "Configuring GCS using a user account",
+ "content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
+ "url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-user-account",
- "relUrl": "/references/"
+ "relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-user-account"
},"469": {
- "doc": "Limitations(v1.2.1)",
- "title": "Velox Backend Limitations",
- "content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
- "url": "/archives/v1.2.1/docs/velox-backend/limitations/#velox-backend-limitations",
+ "doc": "Using GCS with Gluten",
+ "title": "Configuring GCS using a credential file",
+ "content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
+ "url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-credential-file",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/limitations/#velox-backend-limitations"
+ "relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-credential-file"
},"470": {
- "doc": "Limitations(v1.2.1)",
- "title": "Limitations(v1.2.1)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/velox-backend/limitations/",
+ "doc": "Using GCS with Gluten",
+ "title": "Configuring GCS endpoints",
+ "content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
+ "url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-endpoints",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/limitations/"
+ "relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-endpoints"
},"471": {
- "doc": "Limitations",
- "title": "Velox Backend Limitations",
- "content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
- "url": "/docs/velox-backend/limitations/#velox-backend-limitations",
+ "doc": "Using GCS with Gluten",
+ "title": "Configuring GCS max retry count",
+ "content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
+ "url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-count",
- "relUrl": "/docs/velox-backend/limitations/#velox-backend-limitations"
+ "relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-count"
},"472": {
- "doc": "Limitations",
- "title": "Limitations",
- "content": " ",
- "url": "/docs/velox-backend/limitations/",
+ "doc": "Using GCS with Gluten",
+ "title": "Configuring GCS max retry time",
+ "content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retring (10s, 1m, etc). ",
+ "url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-time",
- "relUrl": "/docs/velox-backend/limitations/"
+ "relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-time"
},"473": {
- "doc": "Supported Operators & Functions(v1.2.1)",
- "title": "The Operators and Functions Support Progress",
- "content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. The total number of fun [...]
- "url": "/archives/v1.2.1/docs/velox-backend/support/#the-operators-and-functions-support-progress",
+ "doc": "Using GCS with Gluten",
+ "title": "Using GCS with Gluten",
+ "content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
+ "url": "/archives/v1.3.0/getting-started/gcs/",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/support/#the-operators-and-functions-support-progress"
+ "relUrl": "/archives/v1.3.0/getting-started/gcs/"
},"474": {
- "doc": "Supported Operators & Functions(v1.2.1)",
- "title": "Supported Operators & Functions(v1.2.1)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/velox-backend/support/",
+ "doc": "Velox Local Caching",
+ "title": "Velox Local Caching",
+ "content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
+ "url": "/archives/v1.3.0/getting-started/localcache/",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/support/"
+ "relUrl": "/archives/v1.3.0/getting-started/localcache/"
},"475": {
- "doc": "Supported Operators & Functions",
- "title": "The Operators and Functions Support Progress",
- "content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. The total number of fun [...]
- "url": "/docs/velox-backend/support/#the-operators-and-functions-support-progress",
+ "doc": "Using S3 with Gluten",
+ "title": "Working with S3",
+ "content": " ",
+ "url": "/archives/v1.3.0/getting-started/s3/#working-with-s3",
- "relUrl": "/docs/velox-backend/support/#the-operators-and-functions-support-progress"
+ "relUrl": "/archives/v1.3.0/getting-started/s3/#working-with-s3"
},"476": {
- "doc": "Supported Operators & Functions",
- "title": "Supported Operators & Functions",
- "content": " ",
- "url": "/docs/velox-backend/support/",
+ "doc": "Using S3 with Gluten",
+ "title": "Configuring S3 endpoint",
+ "content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
+ "url": "/archives/v1.3.0/getting-started/s3/#configuring-s3-endpoint",
- "relUrl": "/docs/velox-backend/support/"
+ "relUrl": "/archives/v1.3.0/getting-started/s3/#configuring-s3-endpoint"
},"477": {
- "doc": "Troubleshooting(v1.2.1)",
- "title": "Velox Backend Troubleshooting",
- "content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
- "url": "/archives/v1.2.1/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting",
+ "doc": "Using S3 with Gluten",
+ "title": "Configuring S3 instance credentials",
+ "content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
+ "url": "/archives/v1.3.0/getting-started/s3/#configuring-s3-instance-credentials",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting"
+ "relUrl": "/archives/v1.3.0/getting-started/s3/#configuring-s3-instance-credentials"
},"478": {
- "doc": "Troubleshooting(v1.2.1)",
- "title": "Troubleshooting(v1.2.1)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/velox-backend/troubleshooting/",
+ "doc": "Using S3 with Gluten",
+ "title": "Configuring S3 IAM roles",
+ "content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
+ "url": "/archives/v1.3.0/getting-started/s3/#configuring-s3-iam-roles",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/troubleshooting/"
+ "relUrl": "/archives/v1.3.0/getting-started/s3/#configuring-s3-iam-roles"
},"479": {
- "doc": "Troubleshooting",
- "title": "Velox Backend Troubleshooting",
- "content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
- "url": "/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting",
+ "doc": "Using S3 with Gluten",
+ "title": "Other authentatication methods are not supported yet",
+ "content": ". | AWS temporary credential | . ",
+ "url": "/archives/v1.3.0/getting-started/s3/#other-authentatication-methods-are-not-supported-yet",
- "relUrl": "/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting"
+ "relUrl": "/archives/v1.3.0/getting-started/s3/#other-authentatication-methods-are-not-supported-yet"
},"480": {
- "doc": "Troubleshooting",
- "title": "Troubleshooting",
- "content": " ",
- "url": "/docs/velox-backend/troubleshooting/",
+ "doc": "Using S3 with Gluten",
+ "title": "Log granularity of AWS C++ SDK in velox",
+ "content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
+ "url": "/archives/v1.3.0/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox",
- "relUrl": "/docs/velox-backend/troubleshooting/"
+ "relUrl": "/archives/v1.3.0/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox"
},"481": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
+ "doc": "Using S3 with Gluten",
+ "title": "Local Caching support",
+ "content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
+ "url": "/archives/v1.3.0/getting-started/s3/#local-caching-support",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
+ "relUrl": "/archives/v1.3.0/getting-started/s3/#local-caching-support"
},"482": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Introduction",
- "content": "Velox backend supports User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
- "url": "/archives/v1.2.1/docs/velox-backend/udf/#introduction",
+ "doc": "Using S3 with Gluten",
+ "title": "Using S3 with Gluten",
+ "content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
+ "url": "/archives/v1.3.0/getting-started/s3/",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#introduction"
+ "relUrl": "/archives/v1.3.0/getting-started/s3/"
},"483": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Create and Build UDF/UDAF library",
- "content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
- "url": "/archives/v1.2.1/docs/velox-backend/udf/#create-and-build-udfudaf-library",
+ "doc": "Build Parameters for Velox Backend",
+ "title": "Build Parameters",
+ "content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | OFF | . | build_proto [...]
+ "url": "/archives/v1.3.0/getting-started/build-guide/#build-parameters",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#create-and-build-udfudaf-library"
+ "relUrl": "/archives/v1.3.0/getting-started/build-guide/#build-parameters"
},"484": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Using UDF/UDAF in Gluten",
- "content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the library paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
- "url": "/archives/v1.2.1/docs/velox-backend/udf/#using-udfudaf-in-gluten",
+ "doc": "Build Parameters for Velox Backend",
+ "title": "Gluten Jar for Deployment",
+ "content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . | 3.5.1 | 3.5 | 2.12 | . ",
+ "url": "/archives/v1.3.0/getting-started/build-guide/#gluten-jar-for-deployment",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#using-udfudaf-in-gluten"
+ "relUrl": "/archives/v1.3.0/getting-started/build-guide/#gluten-jar-for-deployment"
},"485": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Try the example",
- "content": "We provided Velox UDF examples in file MyUDF.cc and UDAF examples in file MyUDAF.cc. You need to build the gluten project with --build_example=ON to get the example libraries./dev/buildbundle-veloxbe.sh --build_examples=ON . Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ . Start spark-shell or spark-sql with below configuration . # Use the `--files` option to upload a library and configure its relative path --files /path/to/glute [...]
- "url": "/archives/v1.2.1/docs/velox-backend/udf/#try-the-example",
+ "doc": "Build Parameters for Velox Backend",
+ "title": "Build Parameters for Velox Backend",
+ "content": " ",
+ "url": "/archives/v1.3.0/getting-started/build-guide/",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#try-the-example"
+ "relUrl": "/archives/v1.3.0/getting-started/build-guide/"
},"486": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Configurations",
- "content": "| Parameters | Description | . | spark.gluten.sql.columnar.backend.velox.udfLibraryPaths | Path to the udf/udaf libraries. | . | spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths | Path to the udf/udaf libraries on driver node. Only applicable on yarn-client mode. | . | spark.gluten.sql.columnar.backend.velox.udfAllowTypeConversion | Whether to inject possible cast to convert mismatched data types from input to one registered signatures. | . ",
- "url": "/archives/v1.2.1/docs/velox-backend/udf/#configurations",
+ "doc": "Getting-Started",
+ "title": "Getting Started with Gluten for Apache Spark",
+ "content": " ",
+ "url": "/archives/v1.3.0/getting-started#getting-started-with-gluten-for-apache-spark",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#configurations"
+ "relUrl": "/archives/v1.3.0/getting-started#getting-started-with-gluten-for-apache-spark"
},"487": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Pandas UDFs (a.k.a. Vectorized UDFs)",
+ "doc": "Getting-Started",
+ "title": "Getting-Started",
"content": " ",
- "url": "/archives/v1.2.1/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs",
+ "url": "/archives/v1.3.0/getting-started",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs"
+ "relUrl": "/archives/v1.3.0/getting-started"
},"488": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Introduction",
- "content": "Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer doc. ",
- "url": "/archives/v1.2.1/docs/velox-backend/udf/#introduction-1",
+ "doc": "v1.3.0",
+ "title": "Gluten Documents by version",
+ "content": " ",
+ "url": "/archives/v1.3.0/#gluten-documents-by-version",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#introduction-1"
+ "relUrl": "/archives/v1.3.0/#gluten-documents-by-version"
},"489": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Using Pandas UDFs in Gluten with Velox Backend",
- "content": "Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: . pip3 install pyspark==$SPARK_VERSION cython pip3 install pandas pyarrow . Gluten provides a config to control enable ColumnarArrowEvalPython or not, with true as defalt. spark.gluten.sql.columnar.arrowUdf . Then take following PySpark code for example: . from pyspark.sql.functions import pandas_udf, PandasUDFType import pyspark.sql.functions as [...]
- "url": "/archives/v1.2.1/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend",
+ "doc": "v1.3.0",
+ "title": "v1.3.0",
+ "content": " ",
+ "url": "/archives/v1.3.0/",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend"
+ "relUrl": "/archives/v1.3.0/"
},"490": {
- "doc": "Velox UDF(v1.2.1)",
- "title": "Velox UDF(v1.2.1)",
+ "doc": "Velox Backend",
+ "title": "Gluten Documents by version",
"content": " ",
- "url": "/archives/v1.2.1/docs/velox-backend/udf/",
+ "url": "/archives/v1.3.0/velox-backend#gluten-documents-by-version",
- "relUrl": "/archives/v1.2.1/docs/velox-backend/udf/"
+ "relUrl": "/archives/v1.3.0/velox-backend#gluten-documents-by-version"
},"491": {
+ "doc": "Velox Backend",
+ "title": "Velox Backend",
+ "content": " ",
+ "url": "/archives/v1.3.0/velox-backend",
+
+ "relUrl": "/archives/v1.3.0/velox-backend"
+ },"492": {
+ "doc": "Limitations",
+ "title": "Velox Backend Limitations",
+ "content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
+ "url": "/archives/v1.3.0/velox-backend/limitations/#velox-backend-limitations",
+
+ "relUrl": "/archives/v1.3.0/velox-backend/limitations/#velox-backend-limitations"
+ },"493": {
+ "doc": "Limitations",
+ "title": "Limitations",
+ "content": " ",
+ "url": "/archives/v1.3.0/velox-backend/limitations/",
+
+ "relUrl": "/archives/v1.3.0/velox-backend/limitations/"
+ },"494": {
+ "doc": "Supported Operators & Functions",
+ "title": "The Operators and Functions Support Progress",
+ "content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. The total number of fun [...]
+ "url": "/archives/v1.3.0/velox-backend/support/#the-operators-and-functions-support-progress",
+
+ "relUrl": "/archives/v1.3.0/velox-backend/support/#the-operators-and-functions-support-progress"
+ },"495": {
+ "doc": "Supported Operators & Functions",
+ "title": "Supported Operators & Functions",
+ "content": " ",
+ "url": "/archives/v1.3.0/velox-backend/support/",
+
+ "relUrl": "/archives/v1.3.0/velox-backend/support/"
+ },"496": {
+ "doc": "Troubleshooting",
+ "title": "Velox Backend Troubleshooting",
+ "content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
+ "url": "/archives/v1.3.0/velox-backend/troubleshooting/#velox-backend-troubleshooting",
+
+ "relUrl": "/archives/v1.3.0/velox-backend/troubleshooting/#velox-backend-troubleshooting"
+ },"497": {
+ "doc": "Troubleshooting",
+ "title": "Troubleshooting",
+ "content": " ",
+ "url": "/archives/v1.3.0/velox-backend/troubleshooting/",
+
+ "relUrl": "/archives/v1.3.0/velox-backend/troubleshooting/"
+ },"498": {
"doc": "Velox UDF",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": " ",
- "url": "/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
+ "url": "/archives/v1.3.0/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
- "relUrl": "/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"492": {
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
+ },"499": {
"doc": "Velox UDF",
"title": "Introduction",
"content": "Velox backend supports User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
- "url": "/docs/velox-backend/udf/#introduction",
+ "url": "/archives/v1.3.0/velox-backend/udf/#introduction",
- "relUrl": "/docs/velox-backend/udf/#introduction"
- },"493": {
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/#introduction"
+ },"500": {
"doc": "Velox UDF",
"title": "Create and Build UDF/UDAF library",
"content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
- "url": "/docs/velox-backend/udf/#create-and-build-udfudaf-library",
+ "url": "/archives/v1.3.0/velox-backend/udf/#create-and-build-udfudaf-library",
- "relUrl": "/docs/velox-backend/udf/#create-and-build-udfudaf-library"
- },"494": {
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/#create-and-build-udfudaf-library"
+ },"501": {
"doc": "Velox UDF",
"title": "Using UDF/UDAF in Gluten",
"content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the library paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
- "url": "/docs/velox-backend/udf/#using-udfudaf-in-gluten",
+ "url": "/archives/v1.3.0/velox-backend/udf/#using-udfudaf-in-gluten",
- "relUrl": "/docs/velox-backend/udf/#using-udfudaf-in-gluten"
- },"495": {
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/#using-udfudaf-in-gluten"
+ },"502": {
"doc": "Velox UDF",
"title": "Try the example",
"content": "We provided Velox UDF examples in file MyUDF.cc and UDAF examples in file MyUDAF.cc. You need to build the gluten project with --build_example=ON to get the example libraries./dev/buildbundle-veloxbe.sh --build_examples=ON . Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ . Start spark-shell or spark-sql with below configuration . # Use the `--files` option to upload a library and configure its relative path --files /path/to/glute [...]
- "url": "/docs/velox-backend/udf/#try-the-example",
+ "url": "/archives/v1.3.0/velox-backend/udf/#try-the-example",
- "relUrl": "/docs/velox-backend/udf/#try-the-example"
- },"496": {
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/#try-the-example"
+ },"503": {
"doc": "Velox UDF",
"title": "Configurations",
"content": "| Parameters | Description | . | spark.gluten.sql.columnar.backend.velox.udfLibraryPaths | Path to the udf/udaf libraries. | . | spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths | Path to the udf/udaf libraries on driver node. Only applicable on yarn-client mode. | . | spark.gluten.sql.columnar.backend.velox.udfAllowTypeConversion | Whether to inject possible cast to convert mismatched data types from input to one registered signatures. | . ",
- "url": "/docs/velox-backend/udf/#configurations",
+ "url": "/archives/v1.3.0/velox-backend/udf/#configurations",
- "relUrl": "/docs/velox-backend/udf/#configurations"
- },"497": {
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/#configurations"
+ },"504": {
"doc": "Velox UDF",
"title": "Pandas UDFs (a.k.a. Vectorized UDFs)",
"content": " ",
- "url": "/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs",
+ "url": "/archives/v1.3.0/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs",
- "relUrl": "/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs"
- },"498": {
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs"
+ },"505": {
"doc": "Velox UDF",
"title": "Introduction",
"content": "Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer doc. ",
- "url": "/docs/velox-backend/udf/#introduction-1",
+ "url": "/archives/v1.3.0/velox-backend/udf/#introduction-1",
- "relUrl": "/docs/velox-backend/udf/#introduction-1"
- },"499": {
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/#introduction-1"
+ },"506": {
"doc": "Velox UDF",
"title": "Using Pandas UDFs in Gluten with Velox Backend",
"content": "Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: . pip3 install pyspark==$SPARK_VERSION cython pip3 install pandas pyarrow . Gluten provides a config to control enable ColumnarArrowEvalPython or not, with true as defalt. spark.gluten.sql.columnar.arrowUdf . Then take following PySpark code for example: . from pyspark.sql.functions import pandas_udf, PandasUDFType import pyspark.sql.functions as [...]
- "url": "/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend",
+ "url": "/archives/v1.3.0/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend",
- "relUrl": "/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend"
- },"500": {
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend"
+ },"507": {
"doc": "Velox UDF",
"title": "Velox UDF",
"content": " ",
- "url": "/docs/velox-backend/udf/",
-
- "relUrl": "/docs/velox-backend/udf/"
- },"501": {
- "doc": "Velox Function Development(v1.2.1)",
- "title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
- "content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
- "url": "/archives/v1.2.1/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
-
- "relUrl": "/archives/v1.2.1/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"502": {
- "doc": "Velox Function Development(v1.2.1)",
- "title": "Velox Function Development(v1.2.1)",
- "content": " ",
- "url": "/archives/v1.2.1/docs/developers/velox-function-dev/",
-
- "relUrl": "/archives/v1.2.1/docs/developers/velox-function-dev/"
- },"503": {
- "doc": "v1.1.1/Velox Function Development",
- "title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
- "content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
- "url": "/archives/v1.1.1/developers/velox_function_development/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
-
- "relUrl": "/archives/v1.1.1/developers/velox_function_development/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"504": {
- "doc": "v1.1.1/Velox Function Development",
- "title": "v1.1.1/Velox Function Development",
- "content": " ",
- "url": "/archives/v1.1.1/developers/velox_function_development/",
-
- "relUrl": "/archives/v1.1.1/developers/velox_function_development/"
- },"505": {
- "doc": "Velox Function Development",
- "title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
- "content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
- "url": "/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
-
- "relUrl": "/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"506": {
- "doc": "Velox Function Development",
- "title": "Velox Function Development",
- "content": " ",
- "url": "/docs/developers/velox-function-dev/",
+ "url": "/archives/v1.3.0/velox-backend/udf/",
- "relUrl": "/docs/developers/velox-function-dev/"
+ "relUrl": "/archives/v1.3.0/velox-backend/udf/"
}
}
diff --git a/output/blog/index.html b/output/blog/index.html
index 9143a7b..ebf5686 100644
--- a/output/blog/index.html
+++ b/output/blog/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(5)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(3)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
diff --git a/output/contact/index.html b/output/contact/index.html
index ede1314..0d5752e 100644
--- a/output/contact/index.html
+++ b/output/contact/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(9)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(7)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
diff --git a/output/contributing/index.html b/output/contributing/index.html
index 622aa17..51e9a87 100644
--- a/output/contributing/index.html
+++ b/output/contributing/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(8)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(6)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
diff --git a/output/docs/configuration/index.html b/output/docs/configuration/index.html
deleted file mode 100644
index fb8d650..0000000
--- a/output/docs/configuration/index.html
+++ /dev/null
@@ -1,4 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul.nav-list:first-child > li > ul > li [...]
-<span class="nv">sparkContext</span><span class="o">.</span><span class="py">setLocalProperty</span><span class="o">(</span><span class="n">key</span><span class="o">,</span> <span class="n">value</span><span class="o">)</span>
-<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"select * from demo_tables"</span><span class="o">).</span><span class="py">show</span><span class="o">()</span>
-</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/docs/developers/UsingGperftoolsInCH.html b/output/docs/developers/UsingGperftoolsInCH.html
deleted file mode 100644
index 03da3e7..0000000
--- a/output/docs/developers/UsingGperftoolsInCH.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } </style> <script src="/assets/js/vendor/lunr.min.js"></script> <script src="/assets/js/just-the-docs.js"></script> <meta name= [...]
-export LD_PRELOAD=$tcmalloc_path,libch.so # load the library in the driver
---files $tcmalloc_path # upload the library to the cluster
---conf spark.executorEnv.LD_PRELOAD=./libtcmalloc_and_profiler.so,libch.so # load the library in the executor
---conf spark.executorEnv.HEAPPROFILE=/tmp/gluten_heap_perf # set the heap profile path, you can change to CPUPROFILE for CPU profiling
-</code></pre></div></div> <p>For thrift server on local machine, note using <code class="language-plaintext highlighter-rouge">export LD_PRELOAD="$tcmalloc_path libch.so" # load the library in the driver</code> to preload dynamic libraries.</p> <h2 id="analyze-the-result"> <a href="#analyze-the-result" class="anchor-heading" aria-labelledby="analyze-the-result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Analyze the result </h2> <p>We can get [...]
diff --git a/output/docs/developers/UsingJemallocWithCH.html b/output/docs/developers/UsingJemallocWithCH.html
deleted file mode 100644
index 041f832..0000000
--- a/output/docs/developers/UsingJemallocWithCH.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } </style> <script src="/assets/js/vendor/lunr.min.js"></script> <script src="/assets/js/just-the-docs.js"></script> <meta name= [...]
- # At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracking.
- #
- # ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1).
--target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1)
-+#target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1)
-+target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
- target_link_libraries (_jemalloc PRIVATE unwind)
-</code></pre></div></div> <p>For more infomation, check https://github.com/jemalloc/jemalloc/issues/2282.</p> <h2 id="get-jeprof"> <a href="#get-jeprof" class="anchor-heading" aria-labelledby="get-jeprof"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Get jeprof </h2> <p>Change to the directory where you want to install jemalloc, and run the following commands:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre clas [...]
-</code></pre></div></div> <p>Then we get jeprof in the directory <code class="language-plaintext highlighter-rouge">$Clickhouse_SOURCE_PATH/contrib/jemalloc/bin/jeprof</code>.</p> <h2 id="compiler-libchso"> <a href="#compiler-libchso" class="anchor-heading" aria-labelledby="compiler-libchso"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Compiler libch.so </h2> <p>Ensure to enable jemalloc <code class="language-plaintext highlighter-rouge">-DENAB [...]
-</code></pre></div></div> <p>You can find more options in https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling.</p> <h2 id="analyze-the-result"> <a href="#analyze-the-result" class="anchor-heading" aria-labelledby="analyze-the-result"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Analyze the result </h2> <p>After you get the heap file, you can use the following command to analyze the result: Check memory diff, so you can find whi [...]
-</code></pre></div></div> <p>You can find more usage about jeprof with <code class="language-plaintext highlighter-rouge">jeprof --help</code>.</p> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, [...]
diff --git a/output/docs/developers/cpp-code-style/index.html b/output/docs/developers/cpp-code-style/index.html
deleted file mode 100644
index 58862b9..0000000
--- a/output/docs/developers/cpp-code-style/index.html
+++ /dev/null
@@ -1 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
diff --git a/output/docs/developers/docker-centos7/index.html b/output/docs/developers/docker-centos7/index.html
deleted file mode 100644
index a4e05d5..0000000
--- a/output/docs/developers/docker-centos7/index.html
+++ /dev/null
@@ -1,47 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:7 /bin/bash
-docker attach gluten
-</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum <span class="nt">-y</span> <span class="nb">install </span>epel-release centos-release-scl
-yum <span class="nt">-y</span> <span class="nb">install</span> <span class="se">\</span>
- git236 <span class="se">\</span>
- dnf <span class="se">\</span>
- cmake3 <span class="se">\</span>
- devtoolset-9 <span class="se">\</span>
- java-1.8.0-openjdk <span class="se">\</span>
- java-1.8.0-openjdk-devel <span class="se">\</span>
- ninja-build <span class="se">\</span>
- wget <span class="se">\</span>
- autoconf <span class="se">\</span>
- autoconf-archive <span class="se">\</span>
- automake <span class="se">\</span>
- perl-IPC-Cmd <span class="se">\</span>
- libicu-devel <span class="se">\</span>
- bison <span class="se">\</span>
- libtool <span class="se">\</span>
- patch <span class="se">\</span>
- flex <span class="se">\</span>
- <span class="nb">sudo</span>
-
-<span class="c"># gluten need maven version >=3.6.3</span>
-wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
-<span class="nb">tar</span> <span class="nt">-xvf</span> apache-maven-3.8.8-bin.tar.gz
-<span class="nb">mv </span>apache-maven-3.8.8 /usr/lib/maven
-<span class="nb">export </span><span class="nv">MAVEN_HOME</span><span class="o">=</span>/usr/lib/maven
-<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="k">${</span><span class="nv">PATH</span><span class="k">}</span>:<span class="k">${</span><span class="nv">MAVEN_HOME</span><span class="k">}</span>/bin
-
-<span class="c"># cmake 3.x is required</span>
-<span class="nb">ln</span> <span class="nt">-s</span> /usr/bin/cmake3 /usr/local/bin/cmake
-
-<span class="c"># enable gcc 9</span>
-<span class="nb">.</span> /opt/rh/devtoolset-9/enable <span class="o">||</span> <span class="nb">exit </span>1
-
-<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-1.8.0-openjdk
-<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
-
-git clone https://github.com/oap-project/gluten.git
-<span class="nb">cd </span>gluten
-
-<span class="c"># To access HDFS or S3, you need to add the parameters `--enable_hdfs=ON` and `--enable_s3=ON`</span>
-<span class="c"># If you have the same error with issue-3283, you need to add the parameter `--compile_arrow_java=ON`</span>
-./dev/buildbundle-veloxbe.sh
-</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/docs/developers/docker-centos8/index.html b/output/docs/developers/docker-centos8/index.html
deleted file mode 100644
index fef45f2..0000000
--- a/output/docs/developers/docker-centos8/index.html
+++ /dev/null
@@ -1,35 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:8 /bin/bash
-docker attach gluten
-</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">#update mirror</span>
-<span class="nb">sed</span> <span class="nt">-i</span> <span class="nt">-e</span> <span class="s2">"s|mirrorlist=|#mirrorlist=|g"</span> /etc/yum.repos.d/CentOS-<span class="k">*</span>
-<span class="nb">sed</span> <span class="nt">-i</span> <span class="nt">-e</span> <span class="s2">"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"</span> /etc/yum.repos.d/CentOS-<span class="k">*</span>
-
-dnf <span class="nb">install</span> <span class="nt">-y</span> epel-release <span class="nb">sudo
-</span>yum <span class="nb">install</span> <span class="nt">-y</span> dnf-plugins-core
-yum config-manager <span class="nt">--set-enabled</span> powertools
-dnf <span class="nt">--enablerepo</span><span class="o">=</span>powertools <span class="nb">install</span> <span class="nt">-y</span> ninja-build
-dnf <span class="nt">--enablerepo</span><span class="o">=</span>powertools <span class="nb">install</span> <span class="nt">-y</span> libdwarf-devel
-dnf <span class="nb">install</span> <span class="nt">-y</span> <span class="nt">--setopt</span><span class="o">=</span><span class="nv">install_weak_deps</span><span class="o">=</span>False ccache gcc-toolset-9 git wget which libevent-devel <span class="se">\</span>
- openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel <span class="se">\</span>
- curl-devel cmake libicu-devel
-
-<span class="nb">source</span> /opt/rh/gcc-toolset-9/enable <span class="o">||</span> <span class="nb">exit </span>1
-
-yum <span class="nb">install</span> <span class="nt">-y</span> java-1.8.0-openjdk-devel patch
-<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-1.8.0-openjdk
-<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
-
-<span class="c">#gluten need maven version >=3.6.3</span>
-wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
-<span class="nb">tar</span> <span class="nt">-xvf</span> apache-maven-3.8.8-bin.tar.gz
-<span class="nb">mv </span>apache-maven-3.8.8 /usr/lib/maven
-<span class="nb">export </span><span class="nv">MAVEN_HOME</span><span class="o">=</span>/usr/lib/maven
-<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="k">${</span><span class="nv">PATH</span><span class="k">}</span>:<span class="k">${</span><span class="nv">MAVEN_HOME</span><span class="k">}</span>/bin
-
-git clone https://github.com/oap-project/gluten.git
-<span class="nb">cd </span>gluten
-
-<span class="c"># To access HDFS or S3, you need to add the parameters `--enable_hdfs=ON` and `--enable_s3=ON`</span>
-./dev/buildbundle-veloxbe.sh
-</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/docs/developers/docker-ubuntu/index.html b/output/docs/developers/docker-ubuntu/index.html
deleted file mode 100644
index 5eda128..0000000
--- a/output/docs/developers/docker-ubuntu/index.html
+++ /dev/null
@@ -1,44 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-docker run <span class="nt">-itd</span> <span class="nt">--network</span> host <span class="nt">--name</span> gluten ubuntu:22.04 /bin/bash
-docker attach gluten
-</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
-
-<span class="c">#install gcc and libraries to build arrow</span>
-apt <span class="nb">install </span>software-properties-common
-apt <span class="nb">install </span>maven build-essential cmake libssl-dev libre2-dev libcurl4-openssl-dev clang lldb lld libz-dev git ninja-build uuid-dev autoconf-archive curl zip unzip <span class="nb">tar </span>pkg-config bison libtool flex vim
-
-<span class="c">#velox script needs sudo to install dependency libraries</span>
-apt <span class="nb">install sudo</span>
-
-<span class="c"># make sure jemalloc is uninstalled, jemalloc will be build in vcpkg, which conflicts with the default jemalloc in system</span>
-apt purge libjemalloc-dev libjemalloc2 librust-jemalloc-sys-dev
-
-<span class="c">#make sure jdk8 is used. New version of jdk is not supported</span>
-apt <span class="nb">install</span> <span class="nt">-y</span> openjdk-8-jdk
-apt <span class="nb">install</span> <span class="nt">-y</span> default-jdk
-<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
-<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
-
-<span class="c">#manually install tzdata to avoid the interactive timezone config</span>
-<span class="nb">ln</span> <span class="nt">-fs</span> /usr/share/zoneinfo/America/New_York /etc/localtime
-<span class="nv">DEBIAN_FRONTEND</span><span class="o">=</span>noninteractive apt-get <span class="nb">install</span> <span class="nt">-y</span> tzdata
-dpkg <span class="nt">--configure</span> <span class="nt">-a</span>
-
-<span class="c">#setup proxy on necessary</span>
-<span class="c">#export http_proxy=xxxx</span>
-<span class="c">#export https_proxy=xxxx</span>
-
-<span class="c">#clone gluten</span>
-git clone https://github.com/oap-project/gluten.git
-<span class="nb">cd </span>gluten/
-
-<span class="c">#config maven proxy</span>
-<span class="c">#mkdir ~/.m2/</span>
-<span class="c">#vim ~/.m2/settings.xml</span>
-
-<span class="c"># the script download velox & arrow and compile all dependency library automatically</span>
-<span class="c"># To access HDFS or S3, you need to add the parameters `--enable_hdfs=ON` and `--enable_s3=ON`</span>
-<span class="c"># It's suggested to build using static link, enabled by `--enable_vcpkg=ON`</span>
-<span class="c"># For developer, it's suggested to enable Debug info, by --build_type=RelWithDebInfo. Note RelWithDebInfo uses -o2, release uses -o3</span>
-./dev/buildbundle-veloxbe.sh <span class="nt">--enable_vcpkg</span><span class="o">=</span>ON <span class="nt">--build_type</span><span class="o">=</span>RelWithDebInfo
-</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/docs/developers/how-to-release/index.html b/output/docs/developers/how-to-release/index.html
index 4860eb6..8197670 100644
--- a/output/docs/developers/how-to-release/index.html
+++ b/output/docs/developers/how-to-release/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } .site-nav > ul.nav-category-list > li > button svg { transform: rotate(-90deg); } .site-nav > ul.nav-category-list > li.nav-li [...]
<span class="nv">$ </span>gpg <span class="nt">--full-generate-key</span>
<span class="c"># list the GPG keys</span>
diff --git a/output/docs/developers/how-to-security-scan/index.html b/output/docs/developers/how-to-security-scan/index.html
deleted file mode 100644
index 88b2d6b..0000000
--- a/output/docs/developers/how-to-security-scan/index.html
+++ /dev/null
@@ -1,2 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-</code></pre></div></div> <ol> <li>Open the report file and check if there is any vulnerability issue highlighted. We must guarantee all the vulnerability issue has been solved before an official release.</li> </ol> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Founda [...]
diff --git a/output/docs/developers/how-to-use/index.html b/output/docs/developers/how-to-use/index.html
deleted file mode 100644
index 683d1c3..0000000
--- a/output/docs/developers/how-to-use/index.html
+++ /dev/null
@@ -1,22 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-bash highl [...]
-mvn clean package <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span>
-mvn <span class="nb">test</span> <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span> <span class="nt">-pl</span> backends-velox <span class="nt">-am</span> <span class="nt">-DtagsToInclude</span><span class="o">=</span><span class="s2">"io.glutenproject.tags.GenerateExample"</span> <span class="nt">-Dtest</span><span class="o">=</span>none <span class="nt">-DfailIfNoTests</span><span class="o">=</span><span class="nb">false</span> <s [...]
-</code></pre></div> </div> <ul> <li>After the above operations, the examples files are generated under <code class="language-plaintext highlighter-rouge">gluten_home/backends-velox</code></li> <li>You can check it by the command <code class="language-plaintext highlighter-rouge">tree gluten_home/backends-velox/generated-native-benchmark/</code></li> <li>You may replace <code class="language-plaintext highlighter-rouge">-Pspark-3.2</code> with <code class="language-plaintext highlighter-r [...]
-gluten_home/backends-velox/generated-native-benchmark/
-├── example.json
-├── example_lineitem
-│ ├── part-00000-3ec19189-d20e-4240-85ae-88631d46b612-c000.snappy.parquet
-│ └── _SUCCESS
-└── example_orders
- ├── part-00000-1e66fb98-4dd6-47a6-8679-8625dbc437ee-c000.snappy.parquet
- └── _SUCCESS
-</code></pre></div> </div> </li> </ul> </li> <li>now, run benchmarks with GDB <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">cd </span>gluten_home/cpp/build/velox/benchmarks/
-gdb generic_benchmark
-</code></pre></div> </div> <ul> <li>When GDB load <code class="language-plaintext highlighter-rouge">generic_benchmark</code> successfully, you can set <code class="language-plaintext highlighter-rouge">breakpoint</code> on the <code class="language-plaintext highlighter-rouge">main</code> function with command <code class="language-plaintext highlighter-rouge">b main</code>, and then run with command <code class="language-plaintext highlighter-rouge">r</code>, then the process <code cla [...]
-gdb gluten_home/cpp/build/releases/libgluten.so <span class="s1">'core-Executor task l-2000883-1671542526'</span>
-
-</code></pre></div></div> <ul> <li>the <code class="language-plaintext highlighter-rouge">core-Executor task l-2000883-1671542526</code> represents the core file name.</li> </ul> <h1 id="how-to-run-tpc-h-on-velox-backend"> <a href="#how-to-run-tpc-h-on-velox-backend" class="anchor-heading" aria-labelledby="how-to-run-tpc-h-on-velox-backend"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> How to run TPC-H on Velox backend </h1> <p>Now, both Parquet [...]
-</code></pre></div> </div> </li> <li>set <code class="language-plaintext highlighter-rouge">var gluten_root</code> to correct directory. If <code class="language-plaintext highlighter-rouge">gluten_home</code> is the directory of <code class="language-plaintext highlighter-rouge">/home/gluten</code>, then modify it as below <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="k">var</span> <span class="n">gluten_root</span> <span [...]
-</code></pre></div> </div> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> - Modify `gluten_home/backends-velox/workload/tpch/run_tpch/tpch_parquet.sh`.
-</code></pre></div> </div> </li> <li>Set <code class="language-plaintext highlighter-rouge">GLUTEN_JAR</code> correctly. Please refer to the section of <a href="http://gluten.apache.org/docs/getting-started/velox-backend/#build-gluten-with-velox-backend">Build Gluten with Velox Backend</a></li> <li>Set <code class="language-plaintext highlighter-rouge">SPARK_HOME</code> correctly.</li> <li>Set the memory configurations appropriately. - Execute <code class="language-plaintext highlighter- [...]
diff --git a/output/docs/developers/index.html b/output/docs/developers/index.html
deleted file mode 100644
index ea0f90e..0000000
--- a/output/docs/developers/index.html
+++ /dev/null
@@ -1 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li:not(:nth-child(3)) > a, .site-nav > ul.nav-list:first-child > li > ul > li [...]
diff --git a/output/docs/developers/substrait/index.html b/output/docs/developers/substrait/index.html
deleted file mode 100644
index 82fb3cc..0000000
--- a/output/docs/developers/substrait/index.html
+++ /dev/null
@@ -1 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
diff --git a/output/docs/developers/velox-function-dev/index.html b/output/docs/developers/velox-function-dev/index.html
deleted file mode 100644
index 7670bb0..0000000
--- a/output/docs/developers/velox-function-dev/index.html
+++ /dev/null
@@ -1,11 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-struct BitwiseAndFunction {
- template <typename TInput>
- // For void return type, it indicates null result will never be obtained for non-null input.
- // For bool return type, it indicates null result can be obtained for non-null input (false for null).
- FOLLY_ALWAYS_INLINE void call(TInput& result, TInput a, TInput b) {
- result = a & b;
- }
-};
-</code></pre></div></div> <p>It is templated, as well as the <code class="language-plaintext highlighter-rouge">call</code> function, to allow multiple types. In the above impl., the result will be null for null input. Please use <code class="language-plaintext highlighter-rouge">callNullable</code> if you need different behavior for null input, e.g., get a non-null result for null input. Also see <code class="language-plaintext highlighter-rouge">callNullFree</code> in velox document. I [...]
-</code></pre></div></div> <p>Functions for complex types have similar implementations. See <code class="language-plaintext highlighter-rouge">ArrayAverageFunction</code> in <a href="https://github.com/facebookincubator/velox/blob/main/velox/functions/prestosql/ArrayFunctions.h">velox/functions/prestosql/ArrayFunctions.h</a>.</p> <h3 id="reference"> <a href="#reference" class="anchor-heading" aria-labelledby="reference"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-lin [...]
diff --git a/output/docs/getting-started/abfs/index.html b/output/docs/getting-started/abfs/index.html
deleted file mode 100644
index 3e2d516..0000000
--- a/output/docs/getting-started/abfs/index.html
+++ /dev/null
@@ -1,2 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
diff --git a/output/docs/getting-started/build-guide/index.html b/output/docs/getting-started/build-guide/index.html
deleted file mode 100644
index 58e823c..0000000
--- a/output/docs/getting-started/build-guide/index.html
+++ /dev/null
@@ -1 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
diff --git a/output/docs/getting-started/gcs/index.html b/output/docs/getting-started/gcs/index.html
deleted file mode 100644
index 76e424c..0000000
--- a/output/docs/getting-started/gcs/index.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
-</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage. [...]
-</code></pre></div></div> <h2 id="configuring-gcs-max-retry-count"> <a href="#configuring-gcs-max-retry-count" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-count"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry count </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected.</p> <div class="language-shell highligh [...]
-</code></pre></div></div> <h2 id="configuring-gcs-max-retry-time"> <a href="#configuring-gcs-max-retry-time" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-time"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry time </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration.</p> <div class="language-shell highlig [...]
-</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/docs/getting-started/index.html b/output/docs/getting-started/index.html
deleted file mode 100644
index 9ebd148..0000000
--- a/output/docs/getting-started/index.html
+++ /dev/null
@@ -1 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li:not(:nth-child(1)) > a, .site-nav > ul.nav-list:first-child > li > ul > li [...]
diff --git a/output/docs/getting-started/localcache/index.html b/output/docs/getting-started/localcache/index.html
deleted file mode 100644
index dcf7071..0000000
--- a/output/docs/getting-started/localcache/index.html
+++ /dev/null
@@ -1,8 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-spark.gluten.sql.columnar.backend.velox.memCacheSize // the total size of <span class="k">in</span><span class="nt">-mem</span> cache, default is 128MB.
-spark.gluten.sql.columnar.backend.velox.ssdCachePath // the folder to store the cache files, default is <span class="s2">"/tmp"</span><span class="nb">.</span>
-spark.gluten.sql.columnar.backend.velox.ssdCacheSize // the total size of the SSD cache, default is 128MB. Velox will <span class="k">do in</span><span class="nt">-mem</span> cache only <span class="k">if </span>this value is 0.
-spark.gluten.sql.columnar.backend.velox.ssdCacheShards // the shards of the SSD cache, default is 1.
-spark.gluten.sql.columnar.backend.velox.ssdCacheIOThreads // the IO threads <span class="k">for </span>cache promoting, default is 1. Velox will try to <span class="k">do</span> <span class="s2">"read-ahead"</span> <span class="k">if </span>this value is bigger than 1
-spark.gluten.sql.columnar.backend.velox.ssdODirect // <span class="nb">enable </span>or disable O_DIRECT on cache write, default false.
-</code></pre></div></div> <p>It’s recommended to mount SSDs to the cache path to get the best performance of local caching. Cache files will be written to “spark.gluten.sql.columnar.backend.velox.cachePath”, with UUID based suffix, e.g. “/tmp/cache.13e8ab65-3af4-46ac-8d28-ff99b2a9ec9b0”. Gluten cannot reuse older caches for now, and the old cache files are left after Spark context shutdown.</p> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https:// [...]
diff --git a/output/docs/getting-started/s3/index.html b/output/docs/getting-started/s3/index.html
deleted file mode 100644
index 66e243b..0000000
--- a/output/docs/getting-started/s3/index.html
+++ /dev/null
@@ -1,11 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
-spark.hadoop.fs.s3a.access.key XXXXXXXXX
-spark.hadoop.fs.s3a.secret.key XXXXXXXXX
-spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com
-spark.hadoop.fs.s3a.connection.ssl.enabled <span class="nb">true
-</span>spark.hadoop.fs.s3a.path.style.access <span class="nb">false</span>
-</code></pre></div></div> <h2 id="configuring-s3-instance-credentials"> <a href="#configuring-s3-instance-credentials" class="anchor-heading" aria-labelledby="configuring-s3-instance-credentials"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring S3 instance credentials </h2> <p>S3 also provides other methods for accessing, you can also use instance credentials by setting the following config</p> <div class="language-plaintext highlighter [...]
-</code></pre></div></div> <p>Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation.</p> <h2 id="configuring-s3-iam-roles"> <a href="#configuring-s3-iam-roles" class="anchor-heading" aria-labelledby="configuring-s3-iam-roles"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring S3 IAM roles </h2> <p>You can also use iam role credentials by setting the following c [...]
-spark.hadoop.fs.s3a.iam.role.session.name xxxx
-</code></pre></div></div> <p>Note that <code class="language-plaintext highlighter-rouge">spark.hadoop.fs.s3a.iam.role.session.name</code> is optional.</p> <h2 id="other-authentatication-methods-are-not-supported-yet"> <a href="#other-authentatication-methods-are-not-supported-yet" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-supported-yet"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication met [...]
diff --git a/output/docs/index.html b/output/docs/index.html
deleted file mode 100644
index 2236b06..0000000
--- a/output/docs/index.html
+++ /dev/null
@@ -1 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(2)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
diff --git a/output/docs/latest.html b/output/docs/latest.html
new file mode 100644
index 0000000..a8b48a6
--- /dev/null
+++ b/output/docs/latest.html
@@ -0,0 +1,6 @@
+<p><a href="https://apache.github.io/incubator-gluten/" target="_blank" rel="noopener noreferrer">
+ Click here to view the latest documentation
+</a></p>
+
+<p>If you are not redirected, <a href="https://apache.github.io/incubator-gluten/" target="_blank">click here</a>.</p>
+
diff --git a/output/archives/v1.1.1/developers/UsingGperftoolsInCH.md b/output/docs/v1.1.1/developers/UsingGperftoolsInCH.md
similarity index 100%
rename from output/archives/v1.1.1/developers/UsingGperftoolsInCH.md
rename to output/docs/v1.1.1/developers/UsingGperftoolsInCH.md
diff --git a/output/archives/v1.1.1/developers/UsingJemallocWithCH.md b/output/docs/v1.1.1/developers/UsingJemallocWithCH.md
similarity index 100%
rename from output/archives/v1.1.1/developers/UsingJemallocWithCH.md
rename to output/docs/v1.1.1/developers/UsingJemallocWithCH.md
diff --git a/output/archives/v1.2.1/docs/developers/UsingGperftoolsInCH.md b/output/docs/v1.2.1/docs/developers/UsingGperftoolsInCH.md
similarity index 100%
rename from output/archives/v1.2.1/docs/developers/UsingGperftoolsInCH.md
rename to output/docs/v1.2.1/docs/developers/UsingGperftoolsInCH.md
diff --git a/output/archives/v1.2.1/docs/developers/UsingJemallocWithCH.md b/output/docs/v1.2.1/docs/developers/UsingJemallocWithCH.md
similarity index 100%
rename from output/archives/v1.2.1/docs/developers/UsingJemallocWithCH.md
rename to output/docs/v1.2.1/docs/developers/UsingJemallocWithCH.md
diff --git a/output/archives/v1.2.1/docs/get-started/Work-with-pyspark.ipynb b/output/docs/v1.2.1/docs/get-started/Work-with-pyspark.ipynb
similarity index 100%
rename from output/archives/v1.2.1/docs/get-started/Work-with-pyspark.ipynb
rename to output/docs/v1.2.1/docs/get-started/Work-with-pyspark.ipynb
diff --git a/output/docs/developers/UsingGperftoolsInCH.md b/output/docs/v1.3.0/developers/UsingGperftoolsInCH.md
similarity index 100%
rename from output/docs/developers/UsingGperftoolsInCH.md
rename to output/docs/v1.3.0/developers/UsingGperftoolsInCH.md
diff --git a/output/docs/developers/UsingJemallocWithCH.md b/output/docs/v1.3.0/developers/UsingJemallocWithCH.md
similarity index 100%
rename from output/docs/developers/UsingJemallocWithCH.md
rename to output/docs/v1.3.0/developers/UsingJemallocWithCH.md
diff --git a/output/docs/get-started/Work-with-pyspark.ipynb b/output/docs/v1.3.0/get-started/Work-with-pyspark.ipynb
similarity index 100%
rename from output/docs/get-started/Work-with-pyspark.ipynb
rename to output/docs/v1.3.0/get-started/Work-with-pyspark.ipynb
diff --git a/output/docs/velox-backend/index.html b/output/docs/velox-backend/index.html
deleted file mode 100644
index 2438cc4..0000000
--- a/output/docs/velox-backend/index.html
+++ /dev/null
@@ -1 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li:not(:nth-child(4)) > a, .site-nav > ul.nav-list:first-child > li > ul > li [...]
diff --git a/output/docs/velox-backend/limitations/index.html b/output/docs/velox-backend/limitations/index.html
deleted file mode 100644
index 0f452c4..0000000
--- a/output/docs/velox-backend/limitations/index.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-</code></pre></div></div> <h4 id="filesource-format"> <a href="#filesource-format" class="anchor-heading" aria-labelledby="filesource-format"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> FileSource format </h4> <p>Currently, Gluten only fully supports parquet file format and partially support ORC. If other format is used, scan operator falls back to vanilla spark.</p> <h4 id="partitioned-table-scan"> <a href="#partitioned-table-scan" class="anc [...]
-<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e"</span><span class="o">)</span>
-</code></pre></div></div> <p>Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><spa [...]
- <span class="o">.</span><span class="py">write</span>
- <span class="o">.</span><span class="py">format</span><span class="o">(</span><span class="s">"parquet"</span><span class="o">)</span>
- <span class="o">.</span><span class="py">partitionBy</span><span class="o">(</span><span class="s">"p"</span><span class="o">)</span>
- <span class="o">.</span><span class="py">save</span><span class="o">(</span><span class="nv">f</span><span class="o">.</span><span class="py">getCanonicalPath</span><span class="o">)</span>
-</code></pre></div></div> <h4 id="partition-write-for-spark34-and-later"> <a href="#partition-write-for-spark34-and-later" class="anchor-heading" aria-labelledby="partition-write-for-spark34-and-later"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Partition write (For Spark3.4 and later) </h4> <p>Gluten supports static partition writes and dynamic partition writes.</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre cla [...]
-<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d) SELECT 2 as d, 3 as e"</span><span class="o">)</span>
-</code></pre></div></div> <p>Gluten does not support bucket write, and will fall back to vanilla Spark.</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><span class="o">,</span> <span class= [...]
- <span class="o">.</span><span class="py">write</span>
- <span class="o">.</span><span class="py">format</span><span class="o">(</span><span class="s">"parquet"</span><span class="o">)</span>
- <span class="o">.</span><span class="py">bucketBy</span><span class="o">(</span><span class="mi">2</span><span class="o">,</span> <span class="s">"c1"</span><span class="o">)</span>
- <span class="o">.</span><span class="py">save</span><span class="o">(</span><span class="nv">f</span><span class="o">.</span><span class="py">getCanonicalPath</span><span class="o">)</span>
-</code></pre></div></div> <h4 id="ctas-write-for-spark32-and-spark33"> <a href="#ctas-write-for-spark32-and-spark33" class="anchor-heading" aria-labelledby="ctas-write-for-spark32-and-spark33"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> CTAS write (For Spark3.2 and Spark3.3) </h4> <p>Gluten does not create table as select. It may raise exception. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlig [...]
- <span class="o">.</span><span class="py">write</span>
- <span class="o">.</span><span class="py">format</span><span class="o">(</span><span class="s">"parquet"</span><span class="o">)</span>
- <span class="o">.</span><span class="py">saveAsTable</span><span class="o">(</span><span class="s">"velox_ctas"</span><span class="o">)</span>
-</code></pre></div></div> <h4 id="ctas-write-for-spark34-and-later"> <a href="#ctas-write-for-spark34-and-later" class="anchor-heading" aria-labelledby="ctas-write-for-spark34-and-later"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> CTAS write (For Spark3.4 and later) </h4> <p>Gluten supports create table as select with parquet file format.</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><spa [...]
- <span class="o">.</span><span class="py">write</span>
- <span class="o">.</span><span class="py">format</span><span class="o">(</span><span class="s">"parquet"</span><span class="o">)</span>
- <span class="o">.</span><span class="py">saveAsTable</span><span class="o">(</span><span class="s">"velox_ctas"</span><span class="o">)</span>
-</code></pre></div></div> <h4 id="hivefileformat-write"> <a href="#hivefileformat-write" class="anchor-heading" aria-labelledby="hivefileformat-write"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> HiveFileFormat write </h4> <p>Gluten supports writes of HiveFileFormat when the output file type is of type <code class="language-plaintext highlighter-rouge">parquet</code> only</p> <h4 id="nan-support"> <a href="#nan-support" class="anchor-heading" a [...]
diff --git a/output/docs/velox-backend/support/index.html b/output/docs/velox-backend/support/index.html
deleted file mode 100644
index ba4d08a..0000000
--- a/output/docs/velox-backend/support/index.html
+++ /dev/null
@@ -1 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
diff --git a/output/docs/velox-backend/troubleshooting/index.html b/output/docs/velox-backend/troubleshooting/index.html
deleted file mode 100644
index 1572d68..0000000
--- a/output/docs/velox-backend/troubleshooting/index.html
+++ /dev/null
@@ -1,46 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-# velox backend only
-rm -rf $SPARK_HOME/jars/flatbuffers-*
-rm -rf $SPARK_HOME/jars/arrow-*
-</code></pre></div></div> <h3 id="incompatible-class-error-when-using-native-writer"> <a href="#incompatible-class-error-when-using-native-writer" class="anchor-heading" aria-labelledby="incompatible-class-error-when-using-native-writer"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Incompatible class error when using native writer </h3> <p>Gluten native writer overwrite some vanilla spark classes. Therefore, when running a program that uses glu [...]
-spark.files = {absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
-// The absolute path on running node
-spark.driver.extraClassPath={absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
-// The relative path under the executor working directory
-spark.executor.extraClassPath=./gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
-</code></pre></div></div> <h4 id="configurations-for-yarn-cluster-mode"> <a href="#configurations-for-yarn-cluster-mode" class="anchor-heading" aria-labelledby="configurations-for-yarn-cluster-mode"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configurations for Yarn Cluster mode </h4> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.driver.userClassPathFirst = true
-spark.executor.userClassPathFirst = true
-
-spark.files = {absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
-// The relative path under the executor working directory
-spark.driver.extraClassPath=./gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
-// The relative path under the executor working directory
-spark.executor.extraClassPath=./gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
-</code></pre></div></div> <h4 id="configurations-for-local--standalone-mode"> <a href="#configurations-for-local--standalone-mode" class="anchor-heading" aria-labelledby="configurations-for-local--standalone-mode"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configurations for Local & Standalone mode </h4> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>// The absolute path on running node
-spark.driver.extraClassPath={absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
-// The absolute path on running node
-spark.executor.extraClassPath={absolute_path}/gluten-<spark-version>-<gluten-version>-SNAPSHOT-jar-with-dependencies.jar
-</code></pre></div></div> <h3 id="invalid-pointer-error"> <a href="#invalid-pointer-error" class="anchor-heading" aria-labelledby="invalid-pointer-error"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Invalid pointer error </h3> <p>If the below error is reported at runtime, please re-build gluten with <code class="language-plaintext highlighter-rouge">--compile_arrow_java=ON</code>, then redeploy Gluten jar.</p> <div class="language-plaintext hig [...]
-======= Backtrace: =========
-/lib64/libc.so.6(+0x7d1fd)[0x7f38c29da1fd]
-/lib64/libstdc++.so.6(_ZNSt6locale5_Impl16_M_install_facetEPKNS_2idEPKNS_5facetE+0x142)[0x7f36cb3370d2]
-/lib64/libstdc++.so.6(_ZNSt6locale5_ImplC1Em+0x1e3)[0x7f36cb337523]
-/lib64/libstdc++.so.6(+0x71495)[0x7f36cb338495]
-/lib64/libpthread.so.0(pthread_once+0x50)[0x7f38c3147be0]
-/lib64/libstdc++.so.6(+0x714e1)[0x7f36cb3384e1]
-/lib64/libstdc++.so.6(_ZNSt6localeC2Ev+0x13)[0x7f36cb338523]
-/lib64/libstdc++.so.6(_ZNSt8ios_base4InitC2Ev+0xbc)[0x7f36cb33537c]
-/tmp/jnilib-645156599284574767.tmp(+0x2a90)[0x7f375d235a90]
-/lib64/ld-linux-x86-64.so.2(+0xf4e3)[0x7f38c33664e3]
-/lib64/ld-linux-x86-64.so.2(+0x13b04)[0x7f38c336ab04]
-/lib64/ld-linux-x86-64.so.2(+0xf2f4)[0x7f38c33662f4]
-/lib64/ld-linux-x86-64.so.2(+0x1321b)[0x7f38c336a21b]
-/lib64/libdl.so.2(+0x102b)[0x7f38c2d1f02b]
-/lib64/ld-linux-x86-64.so.2(+0xf2f4)[0x7f38c33662f4]
-/lib64/libdl.so.2(+0x162d)[0x7f38c2d1f62d]
-/lib64/libdl.so.2(dlopen+0x31)[0x7f38c2d1f0c1]
-/usr/local/jdk1.8.0_381/jre/lib/amd64/server/libjvm.so(+0x9292b1)[0x7f38c22732b1]
-/usr/local/jdk1.8.0_381/jre/lib/amd64/server/libjvm.so(JVM_LoadLibrary+0xa1)[0x7f38c205e0c1]
-/usr/local/jdk1.8.0_381/jre/lib/amd64/libjava.so(Java_java_lang_ClassLoader_00024NativeLibrary_load+0x1ac)
-...
-</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/docs/velox-backend/udf/index.html b/output/docs/velox-backend/udf/index.html
deleted file mode 100644
index 4fa539f..0000000
--- a/output/docs/velox-backend/udf/index.html
+++ /dev/null
@@ -1,87 +0,0 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > a, .site-nav > ul.nav-list:first-child > li > ul > li > ul > li:not(:nth- [...]
-
-#include <velox/expression/VectorFunction.h>
-#include <velox/udf/Udf.h>
-
-namespace {
-static const char* kInteger = "integer";
-}
-
-const int kNumMyUdf = 1;
-
-const char* myUdfArgs[] = {kInteger}:
-gluten::UdfEntry myUdfSig = {"myudf", kInteger, 1, myUdfArgs};
-
-class MyUdf : public facebook::velox::exec::VectorFunction {
- ... // Omit concrete implementation
-}
-
-static std::vector<std::shared_ptr<exec::FunctionSignature>>
-myUdfSignatures() {
- return {facebook::velox::exec::FunctionSignatureBuilder()
- .returnType(myUdfSig.dataType)
- .argumentType(myUdfSig.argTypes[0])
- .build()};
-}
-
-DEFINE_GET_NUM_UDF { return kNumMyUdf; }
-
-DEFINE_GET_UDF_ENTRIES { udfEntries[0] = myUdfSig; }
-
-DEFINE_REGISTER_UDF {
- facebook::velox::exec::registerVectorFunction(
- myUdf[0].name, myUdfSignatures(), std::make_unique<MyUdf>());
-}
-
-</code></pre></div> </div> </li> </ul> <p>To build the UDF library, users need to compile the C++ code and link to <code class="language-plaintext highlighter-rouge">libvelox.so</code>. It’s recommended to create a CMakeLists.txt for the project. Here’s an example:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>project(myudf)
-
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-
-set(GLUTEN_HOME /path/to/gluten)
-
-add_library(myudf SHARED "MyUDF.cpp")
-
-find_library(VELOX_LIBRARY REQUIRED NAMES velox HINTS ${GLUTEN_HOME}/cpp/build/releases NO_DEFAULT_PATH)
-
-target_include_directories(myudf PRIVATE ${GLUTEN_HOME}/cpp ${GLUTEN_HOME}/ep/build-velox/build/velox_ep)
-target_link_libraries(myudf PRIVATE ${VELOX_LIBRARY})
-</code></pre></div></div> <p>The steps for creating and building a UDAF library are quite similar to those for a UDF library. The major difference lies in including and defining specific functions within the UDAF header file <a href="../../cpp/velox/udf/Udaf.h">Udaf.h</a></p> <ul> <li><code class="language-plaintext highlighter-rouge">getNumUdaf()</code></li> <li><code class="language-plaintext highlighter-rouge">getUdafEntries(gluten::UdafEntry* udafEntries)</code></li> <li><code class= [...]
-<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>libmyudf.so
-<span class="c"># Needed for Yarn client mode</span>
-<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths<span class="o">=</span>file:///path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so
-</code></pre></div></div> <ul> <li>Use the <code class="language-plaintext highlighter-rouge">--archives</code> option to upload an archive and configure its relative path</li> </ul> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">--archives</span> /path/to/udf_archives.zip#udf_archives
-<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>udf_archives
-<span class="c"># Needed for Yarn client mode</span>
-<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths<span class="o">=</span>file:///path/to/udf_archives.zip
-</code></pre></div></div> <ul> <li>Configure URI</li> </ul> <p>You can also specify the local or HDFS URIs to the UDF libraries or archives. Local URIs should exist on driver and every worker nodes.</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>file:///path/to/library_or_archive
-</code></pre></div></div> <h2 id="try-the-example"> <a href="#try-the-example" class="anchor-heading" aria-labelledby="try-the-example"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Try the example </h2> <p>We provided Velox UDF examples in file <a href="../../cpp/velox/udf/examples/MyUDF.cc">MyUDF.cc</a> and UDAF examples in file <a href="../../cpp/velox/udf/examples/MyUDAF.cc">MyUDAF.cc</a>. You need to build the gluten project with <code clas [...]
-</code></pre></div></div> <p>Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/</p> <p>Start spark-shell or spark-sql with below configuration</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># Use the `--files` option to upload a library and configure its relative path</span>
-<span class="nt">--files</span> /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so
-<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>libmyudf.so
-</code></pre></div></div> <p>or</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># Only configure URI</span>
-<span class="nt">--conf</span> spark.gluten.sql.columnar.backend.velox.udfLibraryPaths<span class="o">=</span>file:///path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so
-</code></pre></div></div> <p>Run query. The functions <code class="language-plaintext highlighter-rouge">myudf1</code> and <code class="language-plaintext highlighter-rouge">myudf2</code> increment the input value by a constant of 5</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>select myudf1(100L), myudf2(1)
-</code></pre></div></div> <p>The output from spark-shell will be like</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>+------------------+----------------+
-|udfexpression(100)|udfexpression(1)|
-+------------------+----------------+
-| 105| 6|
-+------------------+----------------+
-</code></pre></div></div> <h2 id="configurations"> <a href="#configurations" class="anchor-heading" aria-labelledby="configurations"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configurations </h2> <div class="table-wrapper"><table> <thead> <tr> <th>Parameters</th> <th>Description</th> </tr> </thead> <tbody> <tr> <td>spark.gluten.sql.columnar.backend.velox.udfLibraryPaths</td> <td>Path to the udf/udaf libraries.</td> </tr> <tr> <td>spark.glute [...]
-pip3 install pandas pyarrow
-</code></pre></div></div> <p>Gluten provides a config to control enable <code class="language-plaintext highlighter-rouge">ColumnarArrowEvalPython</code> or not, with <code class="language-plaintext highlighter-rouge">true</code> as defalt.</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.gluten.sql.columnar.arrowUdf
-</code></pre></div></div> <p>Then take following <code class="language-plaintext highlighter-rouge">PySpark</code> code for example:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>from pyspark.sql.functions import pandas_udf, PandasUDFType
-import pyspark.sql.functions as F
-import os
-@pandas_udf('long')
-def pandas_plus_one(v):
- return (v + 1)
-df = spark.read.orc("path_to_file").select("quantity").withColumn("processed_quantity", pandas_plus_one("quantity")).select("quantity")
-</code></pre></div></div> <p>The expected physical plan will be:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>== Physical Plan ==
-VeloxColumnarToRowExec
-+- ^(2) ProjectExecTransformer [pythonUDF0#45L AS processed_quantity#41L]
- +- ^(2) InputIteratorTransformer[quantity#2L, pythonUDF0#45L]
- +- ^(2) InputAdapter
- +- ^(2) ColumnarArrowEvalPython [pandas_plus_one(quantity#2L)#40L], [pythonUDF0#45L], 200
- +- ^(1) NativeFileScan orc [quantity#2L] Batched: true, DataFilters: [], Format: ORC, Location: InMemoryFileIndex(1 paths)[file:/***], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<quantity:bigint>
-</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/downloads/index.html b/output/downloads/index.html
index b565761..9ebd55b 100644
--- a/output/downloads/index.html
+++ b/output/downloads/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(4)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(2)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
diff --git a/output/feed.xml b/output/feed.xml
index ef6376e..f0dc0ed 100644
--- a/output/feed.xml
+++ b/output/feed.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="3.9.3">Jekyll</generator><link href="https://gluten.apache.org/feed.xml" rel="self" type="application/atom+xml" /><link href="https://gluten.apache.org/" rel="alternate" type="text/html" /><updated>2025-02-24T10:04:20+00:00</updated><id>https://gluten.apache.org/feed.xml</id><title type="html">Apache Gluten incubating</title><subtitle>Gluten is a middle layer r [...]
+<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="3.9.3">Jekyll</generator><link href="https://gluten.apache.org/feed.xml" rel="self" type="application/atom+xml" /><link href="https://gluten.apache.org/" rel="alternate" type="text/html" /><updated>2025-06-16T10:25:05+00:00</updated><id>https://gluten.apache.org/feed.xml</id><title type="html">Apache Gluten incubating</title><subtitle>Gluten is a middle layer r [...]
<p>What’s new in Gluten 1.2.0</p>
diff --git a/output/index.html b/output/index.html
index 2757803..1b4a089 100644
--- a/output/index.html
+++ b/output/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(1)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(1)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
--master yarn --deploy-mode client \
--conf spark.plugins=org.apache.gluten.GlutenPlugin \
--conf spark.memory.offHeap.enabled=true \
diff --git a/output/jekyll/update/2024/02/08/welcome-to-jekyll.html b/output/jekyll/update/2024/02/08/welcome-to-jekyll.html
index f7a9dad..afbe325 100644
--- a/output/jekyll/update/2024/02/08/welcome-to-jekyll.html
+++ b/output/jekyll/update/2024/02/08/welcome-to-jekyll.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } </style> <script src="/assets/js/vendor/lunr.min.js"></script> <script src="/assets/js/just-the-docs.js"></script> <meta name= [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } .site-nav > ul.nav-category-list > li > button svg { transform: rotate(-90deg); } .site-nav > ul.nav-category-list > li.nav-li [...]
<span class="nb">puts</span> <span class="s2">"Hi, </span><span class="si">#{</span><span class="nb">name</span><span class="si">}</span><span class="s2">"</span>
<span class="k">end</span>
<span class="n">print_hi</span><span class="p">(</span><span class="s1">'Tom'</span><span class="p">)</span>
diff --git a/output/poweredby/index.html b/output/poweredby/index.html
index 7bc3a0f..5fdb9ec 100644
--- a/output/poweredby/index.html
+++ b/output/poweredby/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(7)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(5)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
diff --git a/output/references/index.html b/output/references/index.html
index 93ce409..6b97e31 100644
--- a/output/references/index.html
+++ b/output/references/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(6)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(4)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org