You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by gi...@apache.org on 2026/03/16 07:26:38 UTC
(gluten-site) branch asf-site updated: Automatic Site Publish by Buildbot
This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/gluten-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new e692e73 Automatic Site Publish by Buildbot
e692e73 is described below
commit e692e739fb8a28816f3a21ec0af1763cd8dbf9be
Author: buildbot <us...@infra.apache.org>
AuthorDate: Mon Mar 16 07:26:33 2026 +0000
Automatic Site Publish by Buildbot
---
output/archives/index.html | 2 +-
.../v1.1.1/developers/cpp_code_style/index.html | 2 +-
.../v1.1.1/developers/docker_centos7/index.html | 2 +-
.../archives/v1.1.1/developers/docker_centos8.html | 2 +-
.../v1.1.1/developers/docker_ubuntu/index.html | 2 +-
output/archives/v1.1.1/developers/howto/index.html | 2 +-
.../v1.1.1/developers/howtorelease/index.html | 2 +-
output/archives/v1.1.1/developers/index.html | 2 +-
.../v1.1.1/developers/microbenchmark/index.html | 2 +-
output/archives/v1.1.1/developers/newto/index.html | 2 +-
.../v1.1.1/developers/substrait/index.html | 2 +-
.../velox_function_development/index.html | 2 +-
.../docs/clickhouse/getting-started/index.html | 2 +-
output/archives/v1.1.1/docs/clickhouse/index.html | 2 +-
output/archives/v1.1.1/docs/index.html | 2 +-
.../v1.1.1/docs/velox/build_parameters/index.html | 2 +-
output/archives/v1.1.1/docs/velox/gcs/index.html | 2 +-
.../v1.1.1/docs/velox/getting-started/index.html | 2 +-
output/archives/v1.1.1/docs/velox/index.html | 2 +-
output/archives/v1.1.1/docs/velox/s3/index.html | 2 +-
output/archives/v1.1.1/index.html | 2 +-
.../archives/v1.2.1/docs/configuration/index.html | 2 +-
.../docs/developers/cpp-code-style/index.html | 2 +-
.../docs/developers/docker-centos7/index.html | 2 +-
.../docs/developers/docker-centos8/index.html | 2 +-
.../docs/developers/docker-ubuntu/index.html | 2 +-
.../docs/developers/how-to-release/index.html | 2 +-
.../developers/how-to-security-scan/index.html | 2 +-
.../v1.2.1/docs/developers/how-to-use/index.html | 2 +-
output/archives/v1.2.1/docs/developers/index.html | 2 +-
.../docs/developers/microbenchmarks/index.html | 2 +-
.../v1.2.1/docs/developers/new-to/index.html | 2 +-
.../v1.2.1/docs/developers/substrait/index.html | 2 +-
.../docs/developers/velox-function-dev/index.html | 2 +-
.../v1.2.1/docs/getting-started/abfs/index.html | 2 +-
.../docs/getting-started/build-guide/index.html | 2 +-
.../getting-started/clickhouse-backend/index.html | 2 +-
.../v1.2.1/docs/getting-started/gcs/index.html | 2 +-
.../v1.2.1/docs/getting-started/index.html | 2 +-
.../docs/getting-started/localcache/index.html | 2 +-
.../v1.2.1/docs/getting-started/s3/index.html | 2 +-
.../docs/getting-started/velox-backend/index.html | 2 +-
output/archives/v1.2.1/docs/index.html | 2 +-
.../archives/v1.2.1/docs/velox-backend/index.html | 2 +-
.../docs/velox-backend/limitations/index.html | 2 +-
.../v1.2.1/docs/velox-backend/support/index.html | 2 +-
.../docs/velox-backend/troubleshooting/index.html | 2 +-
.../v1.2.1/docs/velox-backend/udf/index.html | 2 +-
output/archives/v1.2.1/index.html | 2 +-
output/archives/v1.3.0/configuration.html | 2 +-
output/archives/v1.3.0/developers.html | 2 +-
.../v1.3.0/developers/cpp-code-style/index.html | 2 +-
.../v1.3.0/developers/docker-centos7/index.html | 2 +-
.../v1.3.0/developers/docker-centos8/index.html | 2 +-
.../v1.3.0/developers/docker-ubuntu/index.html | 2 +-
.../developers/how-to-security-scan/index.html | 2 +-
.../v1.3.0/developers/how-to-use/index.html | 2 +-
.../v1.3.0/developers/microbenchmarks/index.html | 2 +-
.../archives/v1.3.0/developers/new-to/index.html | 2 +-
.../v1.3.0/developers/substrait/index.html | 2 +-
.../developers/velox-function-dev/index.html | 2 +-
output/archives/v1.3.0/getting-started.html | 2 +-
.../v1.3.0/getting-started/abfs/index.html | 2 +-
.../v1.3.0/getting-started/build-guide/index.html | 2 +-
.../getting-started/clickhouse-backend/index.html | 2 +-
.../archives/v1.3.0/getting-started/gcs/index.html | 2 +-
.../v1.3.0/getting-started/localcache/index.html | 2 +-
.../archives/v1.3.0/getting-started/s3/index.html | 2 +-
.../getting-started/velox-backend/index.html | 2 +-
output/archives/v1.3.0/index.html | 2 +-
output/archives/v1.3.0/velox-backend.html | 2 +-
.../v1.3.0/velox-backend/limitations/index.html | 2 +-
.../v1.3.0/velox-backend/support/index.html | 2 +-
.../velox-backend/troubleshooting/index.html | 2 +-
.../archives/v1.3.0/velox-backend/udf/index.html | 2 +-
output/assets/js/search-data.json | 935 +++++++++++----------
output/docs/developers/how-to-release/index.html | 2 +-
output/feed.xml | 2 +-
output/references/index.html | 2 +-
79 files changed, 549 insertions(+), 542 deletions(-)
diff --git a/output/archives/index.html b/output/archives/index.html
index 2f79718..c245857 100644
--- a/output/archives/index.html
+++ b/output/archives/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li a { background-image: none; } .site-nav > ul:not [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li a { background-image: none; } .site-nav > ul:not [...]
diff --git a/output/archives/v1.1.1/developers/cpp_code_style/index.html b/output/archives/v1.1.1/developers/cpp_code_style/index.html
index 28b0eb2..a2de745 100644
--- a/output/archives/v1.1.1/developers/cpp_code_style/index.html
+++ b/output/archives/v1.1.1/developers/cpp_code_style/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/developers/docker_centos7/index.html b/output/archives/v1.1.1/developers/docker_centos7/index.html
index e1faf27..b611c9c 100644
--- a/output/archives/v1.1.1/developers/docker_centos7/index.html
+++ b/output/archives/v1.1.1/developers/docker_centos7/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run -itd --name gluten centos:7 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum -y install epel-release centos-release-scl
diff --git a/output/archives/v1.1.1/developers/docker_centos8.html b/output/archives/v1.1.1/developers/docker_centos8.html
index ad1aff0..642c551 100644
--- a/output/archives/v1.1.1/developers/docker_centos8.html
+++ b/output/archives/v1.1.1/developers/docker_centos8.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run -itd --name gluten centos:8 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>#update mirror
diff --git a/output/archives/v1.1.1/developers/docker_ubuntu/index.html b/output/archives/v1.1.1/developers/docker_ubuntu/index.html
index 95b70e4..ab24b70 100644
--- a/output/archives/v1.1.1/developers/docker_ubuntu/index.html
+++ b/output/archives/v1.1.1/developers/docker_ubuntu/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
diff --git a/output/archives/v1.1.1/developers/howto/index.html b/output/archives/v1.1.1/developers/howto/index.html
index b0c2025..95180b8 100644
--- a/output/archives/v1.1.1/developers/howto/index.html
+++ b/output/archives/v1.1.1/developers/howto/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-plaintext [...]
mvn clean package -Pspark-3.2 -Pbackends-velox -Prss
mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-velox -am -DtagsToInclude="io.glutenproject.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Darrow.version=11.0.0-gluten -Dexec.skip
diff --git a/output/archives/v1.1.1/developers/howtorelease/index.html b/output/archives/v1.1.1/developers/howtorelease/index.html
index 63eb5f0..c4c8c09 100644
--- a/output/archives/v1.1.1/developers/howtorelease/index.html
+++ b/output/archives/v1.1.1/developers/howtorelease/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
$ gpg --full-generate-key
# list the GPG keys
diff --git a/output/archives/v1.1.1/developers/index.html b/output/archives/v1.1.1/developers/index.html
index d731ee5..6b2bd84 100644
--- a/output/archives/v1.1.1/developers/index.html
+++ b/output/archives/v1.1.1/developers/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/developers/microbenchmark/index.html b/output/archives/v1.1.1/developers/microbenchmark/index.html
index cb64f3f..f635959 100644
--- a/output/archives/v1.1.1/developers/microbenchmark/index.html
+++ b/output/archives/v1.1.1/developers/microbenchmark/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3</span>
diff --git a/output/archives/v1.1.1/developers/newto/index.html b/output/archives/v1.1.1/developers/newto/index.html
index ae452d4..ed3e161 100644
--- a/output/archives/v1.1.1/developers/newto/index.html
+++ b/output/archives/v1.1.1/developers/newto/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/archives/v1.1.1/developers/substrait/index.html b/output/archives/v1.1.1/developers/substrait/index.html
index bac6f1c..2651e4b 100644
--- a/output/archives/v1.1.1/developers/substrait/index.html
+++ b/output/archives/v1.1.1/developers/substrait/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/developers/velox_function_development/index.html b/output/archives/v1.1.1/developers/velox_function_development/index.html
index b050ab1..89867e7 100644
--- a/output/archives/v1.1.1/developers/velox_function_development/index.html
+++ b/output/archives/v1.1.1/developers/velox_function_development/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html b/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html
index 33bdeaa..c6c9960 100644
--- a/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html
+++ b/output/archives/v1.1.1/docs/clickhouse/getting-started/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <p>If you don’t care about development environment, you can skip this part.</p> <p>Otherwise, do:</p> <ol> <li>cl [...]
git clone --recursive --shallow-submodules -b clickhouse_backend https://github.com/Kyligence/ClickHouse.git
</code></pre></div> </div> </li> <li> <p>Configure cpp-ch ${GLUTEN_SOURCE}/cpp-ch can be treated as an add-on of Kyligence/Clickhouse</p> <p>First, initialize some configuration for this add-on:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="nb">export </span><span class="nv">GLUTEN_SOURCE</span><span class="o">=</span>/path/to/gluten
diff --git a/output/archives/v1.1.1/docs/clickhouse/index.html b/output/archives/v1.1.1/docs/clickhouse/index.html
index f38a448..02c1f87 100644
--- a/output/archives/v1.1.1/docs/clickhouse/index.html
+++ b/output/archives/v1.1.1/docs/clickhouse/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/index.html b/output/archives/v1.1.1/docs/index.html
index dd9c367..d3ebd34 100644
--- a/output/archives/v1.1.1/docs/index.html
+++ b/output/archives/v1.1.1/docs/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/velox/build_parameters/index.html b/output/archives/v1.1.1/docs/velox/build_parameters/index.html
index d6d01d5..6df2891 100644
--- a/output/archives/v1.1.1/docs/velox/build_parameters/index.html
+++ b/output/archives/v1.1.1/docs/velox/build_parameters/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/velox/gcs/index.html b/output/archives/v1.1.1/docs/velox/gcs/index.html
index bfe7144..6f2429d 100644
--- a/output/archives/v1.1.1/docs/velox/gcs/index.html
+++ b/output/archives/v1.1.1/docs/velox/gcs/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-sh highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage.roo [...]
</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.1.1/docs/velox/getting-started/index.html b/output/archives/v1.1.1/docs/velox/getting-started/index.html
index eddf049..c75a085 100644
--- a/output/archives/v1.1.1/docs/velox/getting-started/index.html
+++ b/output/archives/v1.1.1/docs/velox/getting-started/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
diff --git a/output/archives/v1.1.1/docs/velox/index.html b/output/archives/v1.1.1/docs/velox/index.html
index 474cb7b..8fbdab0 100644
--- a/output/archives/v1.1.1/docs/velox/index.html
+++ b/output/archives/v1.1.1/docs/velox/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.1.1/docs/velox/s3/index.html b/output/archives/v1.1.1/docs/velox/s3/index.html
index b5b64e2..44f8dfb 100644
--- a/output/archives/v1.1.1/docs/velox/s3/index.html
+++ b/output/archives/v1.1.1/docs/velox/s3/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
diff --git a/output/archives/v1.1.1/index.html b/output/archives/v1.1.1/index.html
index f209d33..211d196 100644
--- a/output/archives/v1.1.1/index.html
+++ b/output/archives/v1.1.1/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(1)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(1)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.2.1/docs/configuration/index.html b/output/archives/v1.2.1/docs/configuration/index.html
index b4366c7..63c87e3 100644
--- a/output/archives/v1.2.1/docs/configuration/index.html
+++ b/output/archives/v1.2.1/docs/configuration/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nv">sparkContext</span><span class="o">.</span><span class="py">setLocalProperty</span><span class="o">(</span><span class="n">key</span><span class="o">,</span> <span class="n">value</span><span class="o">)</span>
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"select * from demo_tables"</span><span class="o">).</span><span class="py">show</span><span class="o">()</span>
</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html b/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html
index fb6c155..4ac4c06 100644
--- a/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html
+++ b/output/archives/v1.2.1/docs/developers/cpp-code-style/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/developers/docker-centos7/index.html b/output/archives/v1.2.1/docs/developers/docker-centos7/index.html
index 406f2fb..8a270b5 100644
--- a/output/archives/v1.2.1/docs/developers/docker-centos7/index.html
+++ b/output/archives/v1.2.1/docs/developers/docker-centos7/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:7 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum <span class="nt">-y</span> <span class="nb">install </span>epel-release centos-release-scl
diff --git a/output/archives/v1.2.1/docs/developers/docker-centos8/index.html b/output/archives/v1.2.1/docs/developers/docker-centos8/index.html
index 469c930..36fa204 100644
--- a/output/archives/v1.2.1/docs/developers/docker-centos8/index.html
+++ b/output/archives/v1.2.1/docs/developers/docker-centos8/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:8 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">#update mirror</span>
diff --git a/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html b/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html
index 3869bfb..a86583d 100644
--- a/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html
+++ b/output/archives/v1.2.1/docs/developers/docker-ubuntu/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--network</span> host <span class="nt">--name</span> gluten ubuntu:22.04 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
diff --git a/output/archives/v1.2.1/docs/developers/how-to-release/index.html b/output/archives/v1.2.1/docs/developers/how-to-release/index.html
index fc2183a..c8601ae 100644
--- a/output/archives/v1.2.1/docs/developers/how-to-release/index.html
+++ b/output/archives/v1.2.1/docs/developers/how-to-release/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nv">$ </span>gpg <span class="nt">--full-generate-key</span>
<span class="c"># list the GPG keys</span>
diff --git a/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html b/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html
index 288b9e1..acce3f2 100644
--- a/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html
+++ b/output/archives/v1.2.1/docs/developers/how-to-security-scan/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <ol> <li>Open the report file and check if there is any vulnerability issue highlighted. We must guarantee all the vulnerability issue has been solved before an official release.</li> </ol> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Founda [...]
diff --git a/output/archives/v1.2.1/docs/developers/how-to-use/index.html b/output/archives/v1.2.1/docs/developers/how-to-use/index.html
index 68438e7..f3363ff 100644
--- a/output/archives/v1.2.1/docs/developers/how-to-use/index.html
+++ b/output/archives/v1.2.1/docs/developers/how-to-use/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-bash highl [...]
mvn clean package <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span>
mvn <span class="nb">test</span> <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span> <span class="nt">-pl</span> backends-velox <span class="nt">-am</span> <span class="nt">-DtagsToInclude</span><span class="o">=</span><span class="s2">"io.glutenproject.tags.GenerateExample"</span> <span class="nt">-Dtest</span><span class="o">=</span>none <span class="nt">-DfailIfNoTests</span><span class="o">=</span><span class="nb">false</span> <s [...]
diff --git a/output/archives/v1.2.1/docs/developers/index.html b/output/archives/v1.2.1/docs/developers/index.html
index abb46b1..6790ef5 100644
--- a/output/archives/v1.2.1/docs/developers/index.html
+++ b/output/archives/v1.2.1/docs/developers/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html b/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html
index 4b81305..9920bb0 100644
--- a/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html
+++ b/output/archives/v1.2.1/docs/developers/microbenchmarks/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3</span>
diff --git a/output/archives/v1.2.1/docs/developers/new-to/index.html b/output/archives/v1.2.1/docs/developers/new-to/index.html
index 77a2003..6e9ab5c 100644
--- a/output/archives/v1.2.1/docs/developers/new-to/index.html
+++ b/output/archives/v1.2.1/docs/developers/new-to/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/archives/v1.2.1/docs/developers/substrait/index.html b/output/archives/v1.2.1/docs/developers/substrait/index.html
index 72bf82b..01e9b3c 100644
--- a/output/archives/v1.2.1/docs/developers/substrait/index.html
+++ b/output/archives/v1.2.1/docs/developers/substrait/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html b/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html
index 988fa5c..5ec591e 100644
--- a/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html
+++ b/output/archives/v1.2.1/docs/developers/velox-function-dev/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/archives/v1.2.1/docs/getting-started/abfs/index.html b/output/archives/v1.2.1/docs/getting-started/abfs/index.html
index b70b6c3..94aec76 100644
--- a/output/archives/v1.2.1/docs/getting-started/abfs/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/abfs/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/build-guide/index.html b/output/archives/v1.2.1/docs/getting-started/build-guide/index.html
index ff0e952..c0b6ab0 100644
--- a/output/archives/v1.2.1/docs/getting-started/build-guide/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/build-guide/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html b/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html
index f310911..dc91b09 100644
--- a/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/clickhouse-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <p>If you don’t care about development environment, you can skip this part.</p> <p>Otherwise, do:</p> <ol> <li>cl [...]
git clone <span class="nt">--recursive</span> <span class="nt">--shallow-submodules</span> <span class="nt">-b</span> clickhouse_backend https://github.com/Kyligence/ClickHouse.git
</code></pre></div> </div> </li> <li> <p>Configure cpp-ch ${GLUTEN_SOURCE}/cpp-ch can be treated as an add-on of Kyligence/Clickhouse</p> <p>First, initialize some configuration for this add-on:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="nb">export </span><span class="nv">GLUTEN_SOURCE</span><span class="o">=</span>/path/to/gluten
diff --git a/output/archives/v1.2.1/docs/getting-started/gcs/index.html b/output/archives/v1.2.1/docs/getting-started/gcs/index.html
index 6e22e1d..6be7c96 100644
--- a/output/archives/v1.2.1/docs/getting-started/gcs/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/gcs/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage. [...]
</code></pre></div></div> <h2 id="configuring-gcs-max-retry-count"> <a href="#configuring-gcs-max-retry-count" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-count"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry count </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected.</p> <div class="language-shell highligh [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/index.html b/output/archives/v1.2.1/docs/getting-started/index.html
index 1f61452..f93e895 100644
--- a/output/archives/v1.2.1/docs/getting-started/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/getting-started/localcache/index.html b/output/archives/v1.2.1/docs/getting-started/localcache/index.html
index d384e03..a218bee 100644
--- a/output/archives/v1.2.1/docs/getting-started/localcache/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/localcache/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.gluten.sql.columnar.backend.velox.memCacheSize // the total size of <span class="k">in</span><span class="nt">-mem</span> cache, default is 128MB.
spark.gluten.sql.columnar.backend.velox.ssdCachePath // the folder to store the cache files, default is <span class="s2">"/tmp"</span><span class="nb">.</span>
spark.gluten.sql.columnar.backend.velox.ssdCacheSize // the total size of the SSD cache, default is 128MB. Velox will <span class="k">do in</span><span class="nt">-mem</span> cache only <span class="k">if </span>this value is 0.
diff --git a/output/archives/v1.2.1/docs/getting-started/s3/index.html b/output/archives/v1.2.1/docs/getting-started/s3/index.html
index 234eaa6..a2e7ee1 100644
--- a/output/archives/v1.2.1/docs/getting-started/s3/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/s3/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
diff --git a/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html b/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html
index d7f5c6d..7d4493c 100644
--- a/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html
+++ b/output/archives/v1.2.1/docs/getting-started/velox-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
diff --git a/output/archives/v1.2.1/docs/index.html b/output/archives/v1.2.1/docs/index.html
index 8ab0185..1e5c7ec 100644
--- a/output/archives/v1.2.1/docs/index.html
+++ b/output/archives/v1.2.1/docs/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/index.html b/output/archives/v1.2.1/docs/velox-backend/index.html
index 7372621..3d455bd 100644
--- a/output/archives/v1.2.1/docs/velox-backend/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/limitations/index.html b/output/archives/v1.2.1/docs/velox-backend/limitations/index.html
index b35b80d..e22b0ba 100644
--- a/output/archives/v1.2.1/docs/velox-backend/limitations/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/limitations/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="filesource-format"> <a href="#filesource-format" class="anchor-heading" aria-labelledby="filesource-format"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> FileSource format </h4> <p>Currently, Gluten only fully supports parquet file format and partially support ORC. If other format is used, scan operator falls back to vanilla spark.</p> <h4 id="partitioned-table-scan"> <a href="#partitioned-table-scan" class="anc [...]
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e"</span><span class="o">)</span>
</code></pre></div></div> <p>Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><spa [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/support/index.html b/output/archives/v1.2.1/docs/velox-backend/support/index.html
index 3ce9e8c..7cb53f5 100644
--- a/output/archives/v1.2.1/docs/velox-backend/support/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/support/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html b/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html
index cf4db0d..ff714b3 100644
--- a/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/troubleshooting/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
# velox backend only
rm -rf $SPARK_HOME/jars/flatbuffers-*
rm -rf $SPARK_HOME/jars/arrow-*
diff --git a/output/archives/v1.2.1/docs/velox-backend/udf/index.html b/output/archives/v1.2.1/docs/velox-backend/udf/index.html
index 1ec5c48..6603097 100644
--- a/output/archives/v1.2.1/docs/velox-backend/udf/index.html
+++ b/output/archives/v1.2.1/docs/velox-backend/udf/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
#include <velox/expression/VectorFunction.h>
#include <velox/udf/Udf.h>
diff --git a/output/archives/v1.2.1/index.html b/output/archives/v1.2.1/index.html
index 45311ab..0231f08 100644
--- a/output/archives/v1.2.1/index.html
+++ b/output/archives/v1.2.1/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(2)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.3.0/configuration.html b/output/archives/v1.3.0/configuration.html
index a953e50..769e6ef 100644
--- a/output/archives/v1.3.0/configuration.html
+++ b/output/archives/v1.3.0/configuration.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nv">sparkContext</span><span class="o">.</span><span class="py">setLocalProperty</span><span class="o">(</span><span class="n">key</span><span class="o">,</span> <span class="n">value</span><span class="o">)</span>
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"select * from demo_tables"</span><span class="o">).</span><span class="py">show</span><span class="o">()</span>
</code></pre></div></div> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. Apache Gluten, Gluten, Apache, the Apache feather logo, and the Apache Gluten project logo are either registered trademarks or trademark [...]
diff --git a/output/archives/v1.3.0/developers.html b/output/archives/v1.3.0/developers.html
index 22665ab..830589a 100644
--- a/output/archives/v1.3.0/developers.html
+++ b/output/archives/v1.3.0/developers.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/developers/cpp-code-style/index.html b/output/archives/v1.3.0/developers/cpp-code-style/index.html
index 16e5cea..d1e9a42 100644
--- a/output/archives/v1.3.0/developers/cpp-code-style/index.html
+++ b/output/archives/v1.3.0/developers/cpp-code-style/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/developers/docker-centos7/index.html b/output/archives/v1.3.0/developers/docker-centos7/index.html
index 40efbbb..ae70f1b 100644
--- a/output/archives/v1.3.0/developers/docker-centos7/index.html
+++ b/output/archives/v1.3.0/developers/docker-centos7/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:7 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>yum <span class="nt">-y</span> <span class="nb">install </span>epel-release centos-release-scl
diff --git a/output/archives/v1.3.0/developers/docker-centos8/index.html b/output/archives/v1.3.0/developers/docker-centos8/index.html
index 0dbfaa0..654dd69 100644
--- a/output/archives/v1.3.0/developers/docker-centos8/index.html
+++ b/output/archives/v1.3.0/developers/docker-centos8/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--name</span> gluten centos:8 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">#update mirror</span>
diff --git a/output/archives/v1.3.0/developers/docker-ubuntu/index.html b/output/archives/v1.3.0/developers/docker-ubuntu/index.html
index 6b897ba..8294631 100644
--- a/output/archives/v1.3.0/developers/docker-ubuntu/index.html
+++ b/output/archives/v1.3.0/developers/docker-ubuntu/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
docker run <span class="nt">-itd</span> <span class="nt">--network</span> host <span class="nt">--name</span> gluten ubuntu:22.04 /bin/bash
docker attach gluten
</code></pre></div></div> <p>Run in docker:</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update
diff --git a/output/archives/v1.3.0/developers/how-to-security-scan/index.html b/output/archives/v1.3.0/developers/how-to-security-scan/index.html
index 766fae7..6adcd8f 100644
--- a/output/archives/v1.3.0/developers/how-to-security-scan/index.html
+++ b/output/archives/v1.3.0/developers/how-to-security-scan/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <ol> <li>Open the report file and check if there is any vulnerability issue highlighted. We must guarantee all the vulnerability issue has been solved before an official release.</li> </ol> </main> <hr> <footer> <p><a href="#top" id="back-to-top">Back to top</a></p> <a href="https://incubator.apache.org/"><img src="/assets/images/apache-incubator.svg" style="width:300px"></a> <p class="text-small text-grey-dk-100 mb-0">Copyright © 2024 The Apache Software Founda [...]
diff --git a/output/archives/v1.3.0/developers/how-to-use/index.html b/output/archives/v1.3.0/developers/how-to-use/index.html
index 9d10d93..a3a4928 100644
--- a/output/archives/v1.3.0/developers/how-to-use/index.html
+++ b/output/archives/v1.3.0/developers/how-to-use/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div> </div> <ul> <li>Compiling with <code class="language-plaintext highlighter-rouge">--build_type=Debug</code> is good for debugging.</li> <li>The executable file <code class="language-plaintext highlighter-rouge">generic_benchmark</code> will be generated under the directory of <code class="language-plaintext highlighter-rouge">gluten_home/cpp/build/velox/benchmarks/</code>.</li> </ul> </li> <li>build Gluten and generate the example files <div class="language-bash highl [...]
mvn clean package <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span>
mvn <span class="nb">test</span> <span class="nt">-Pspark-3</span>.2 <span class="nt">-Pbackends-velox</span> <span class="nt">-Prss</span> <span class="nt">-pl</span> backends-velox <span class="nt">-am</span> <span class="nt">-DtagsToInclude</span><span class="o">=</span><span class="s2">"io.glutenproject.tags.GenerateExample"</span> <span class="nt">-Dtest</span><span class="o">=</span>none <span class="nt">-DfailIfNoTests</span><span class="o">=</span><span class="nb">false</span> <s [...]
diff --git a/output/archives/v1.3.0/developers/microbenchmarks/index.html b/output/archives/v1.3.0/developers/microbenchmarks/index.html
index 9956cd4..fe316cb 100644
--- a/output/archives/v1.3.0/developers/microbenchmarks/index.html
+++ b/output/archives/v1.3.0/developers/microbenchmarks/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
./dev/buildbundle-veloxbe.sh <span class="nt">--build_tests</span><span class="o">=</span>ON <span class="nt">--build_benchmarks</span><span class="o">=</span>ON
<span class="c"># Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3</span>
diff --git a/output/archives/v1.3.0/developers/new-to/index.html b/output/archives/v1.3.0/developers/new-to/index.html
index f1c2da7..97ecfe0 100644
--- a/output/archives/v1.3.0/developers/new-to/index.html
+++ b/output/archives/v1.3.0/developers/new-to/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
update-alternatives <span class="nt">--config</span> java
java <span class="nt">-version</span>
</code></pre></div></div> <p><code class="language-plaintext highlighter-rouge">--config java</code> to config java executable path, <code class="language-plaintext highlighter-rouge">javac</code> and other commands can also use this command to config. For some other uses, we suggest to set <code class="language-plaintext highlighter-rouge">JAVA_HOME</code>.</p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span [...]
diff --git a/output/archives/v1.3.0/developers/substrait/index.html b/output/archives/v1.3.0/developers/substrait/index.html
index b1bf4de..1d3353d 100644
--- a/output/archives/v1.3.0/developers/substrait/index.html
+++ b/output/archives/v1.3.0/developers/substrait/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/developers/velox-function-dev/index.html b/output/archives/v1.3.0/developers/velox-function-dev/index.html
index c78d443..b4a0135 100644
--- a/output/archives/v1.3.0/developers/velox-function-dev/index.html
+++ b/output/archives/v1.3.0/developers/velox-function-dev/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
struct BitwiseAndFunction {
template <typename TInput>
// For void return type, it indicates null result will never be obtained for non-null input.
diff --git a/output/archives/v1.3.0/getting-started.html b/output/archives/v1.3.0/getting-started.html
index 0afa218..697cc8e 100644
--- a/output/archives/v1.3.0/getting-started.html
+++ b/output/archives/v1.3.0/getting-started.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/getting-started/abfs/index.html b/output/archives/v1.3.0/getting-started/abfs/index.html
index ca3c3f7..e7639bd 100644
--- a/output/archives/v1.3.0/getting-started/abfs/index.html
+++ b/output/archives/v1.3.0/getting-started/abfs/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h3 id="other-authentatication-methods-are-not-yet-supported"> <a href="#other-authentatication-methods-are-not-yet-supported" class="anchor-heading" aria-labelledby="other-authentatication-methods-are-not-yet-supported"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Other authentatication methods are not yet supported. </h3> <h1 id="local-caching-support"> <a href="#local-caching-support" class="anchor-heading" aria-lab [...]
diff --git a/output/archives/v1.3.0/getting-started/build-guide/index.html b/output/archives/v1.3.0/getting-started/build-guide/index.html
index 42f0bf3..05ed2d8 100644
--- a/output/archives/v1.3.0/getting-started/build-guide/index.html
+++ b/output/archives/v1.3.0/getting-started/build-guide/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html b/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html
index 4b2bf1c..0508b34 100644
--- a/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html
+++ b/output/archives/v1.3.0/getting-started/clickhouse-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="setup-clickhouse-backend-development-environment"> <a href="#setup-clickhouse-backend-development-environment" class="anchor-heading" aria-labelledby="setup-clickhouse-backend-development-environment"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Setup ClickHouse backend development environment </h4> <p>If you don’t care about development environment, you can skip this part.</p> <p>Otherwise, do:</p> <ol> <li>cl [...]
git clone <span class="nt">--recursive</span> <span class="nt">--shallow-submodules</span> <span class="nt">-b</span> clickhouse_backend https://github.com/Kyligence/ClickHouse.git
</code></pre></div> </div> </li> <li> <p>Configure cpp-ch ${GLUTEN_SOURCE}/cpp-ch can be treated as an add-on of Kyligence/Clickhouse</p> <p>First, initialize some configuration for this add-on:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code> <span class="nb">export </span><span class="nv">GLUTEN_SOURCE</span><span class="o">=</span>/path/to/gluten
diff --git a/output/archives/v1.3.0/getting-started/gcs/index.html b/output/archives/v1.3.0/getting-started/gcs/index.html
index 0dc1aed..7b79734 100644
--- a/output/archives/v1.3.0/getting-started/gcs/index.html
+++ b/output/archives/v1.3.0/getting-started/gcs/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file with the credentials.
</code></pre></div></div> <h2 id="configuring-gcs-endpoints"> <a href="#configuring-gcs-endpoints" class="anchor-heading" aria-labelledby="configuring-gcs-endpoints"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS endpoints </h2> <p>For cases when a GCS mock is used, an optional endpoint can be provided:</p> <div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>spark.hadoop.fs.gs.storage. [...]
</code></pre></div></div> <h2 id="configuring-gcs-max-retry-count"> <a href="#configuring-gcs-max-retry-count" class="anchor-heading" aria-labelledby="configuring-gcs-max-retry-count"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> Configuring GCS max retry count </h2> <p>For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected.</p> <div class="language-shell highligh [...]
diff --git a/output/archives/v1.3.0/getting-started/localcache/index.html b/output/archives/v1.3.0/getting-started/localcache/index.html
index cddf780..0526f2b 100644
--- a/output/archives/v1.3.0/getting-started/localcache/index.html
+++ b/output/archives/v1.3.0/getting-started/localcache/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.gluten.sql.columnar.backend.velox.memCacheSize // the total size of <span class="k">in</span><span class="nt">-mem</span> cache, default is 128MB.
spark.gluten.sql.columnar.backend.velox.ssdCachePath // the folder to store the cache files, default is <span class="s2">"/tmp"</span><span class="nb">.</span>
spark.gluten.sql.columnar.backend.velox.ssdCacheSize // the total size of the SSD cache, default is 128MB. Velox will <span class="k">do in</span><span class="nt">-mem</span> cache only <span class="k">if </span>this value is 0.
diff --git a/output/archives/v1.3.0/getting-started/s3/index.html b/output/archives/v1.3.0/getting-started/s3/index.html
index fc6ef9f..bc4b25f 100644
--- a/output/archives/v1.3.0/getting-started/s3/index.html
+++ b/output/archives/v1.3.0/getting-started/s3/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.access.key XXXXXXXXX
spark.hadoop.fs.s3a.secret.key XXXXXXXXX
diff --git a/output/archives/v1.3.0/getting-started/velox-backend/index.html b/output/archives/v1.3.0/getting-started/velox-backend/index.html
index 4d8b6f4..11f5415 100644
--- a/output/archives/v1.3.0/getting-started/velox-backend/index.html
+++ b/output/archives/v1.3.0/getting-started/velox-backend/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
<span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/java-8-openjdk-amd64
<span class="nb">export </span><span class="nv">PATH</span><span class="o">=</span><span class="nv">$JAVA_HOME</span>/bin:<span class="nv">$PATH</span>
</code></pre></div></div> <p><strong>For aarch64</strong></p> <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c">## make sure jdk8 is used</span>
diff --git a/output/archives/v1.3.0/index.html b/output/archives/v1.3.0/index.html
index 1ae4eac..a15a596 100644
--- a/output/archives/v1.3.0/index.html
+++ b/output/archives/v1.3.0/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(3)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li:not(:nth-child(3)) > a, .site-nav > ul:nth-of-type(2) > li > ul > l [...]
diff --git a/output/archives/v1.3.0/velox-backend.html b/output/archives/v1.3.0/velox-backend.html
index 67cd7b9..3ca590c 100644
--- a/output/archives/v1.3.0/velox-backend.html
+++ b/output/archives/v1.3.0/velox-backend.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/velox-backend/limitations/index.html b/output/archives/v1.3.0/velox-backend/limitations/index.html
index 576c13c..f881e77 100644
--- a/output/archives/v1.3.0/velox-backend/limitations/index.html
+++ b/output/archives/v1.3.0/velox-backend/limitations/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
</code></pre></div></div> <h4 id="filesource-format"> <a href="#filesource-format" class="anchor-heading" aria-labelledby="filesource-format"><svg viewBox="0 0 16 16" aria-hidden="true"><use xlink:href="#svg-link"></use></svg></a> FileSource format </h4> <p>Currently, Gluten only fully supports parquet file format and partially support ORC. If other format is used, scan operator falls back to vanilla spark.</p> <h4 id="partitioned-table-scan"> <a href="#partitioned-table-scan" class="anc [...]
<span class="nv">spark</span><span class="o">.</span><span class="py">sql</span><span class="o">(</span><span class="s">"INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e"</span><span class="o">)</span>
</code></pre></div></div> <p>Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g.,</p> <div class="language-scala highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">spark</span><span class="o">.</span><span class="py">range</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="py">selectExpr</span><span class="o">(</span><span class="s">"id as c1"</span><spa [...]
diff --git a/output/archives/v1.3.0/velox-backend/support/index.html b/output/archives/v1.3.0/velox-backend/support/index.html
index 8e46e6d..272bf9b 100644
--- a/output/archives/v1.3.0/velox-backend/support/index.html
+++ b/output/archives/v1.3.0/velox-backend/support/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
diff --git a/output/archives/v1.3.0/velox-backend/troubleshooting/index.html b/output/archives/v1.3.0/velox-backend/troubleshooting/index.html
index bba2e0f..dca1fcb 100644
--- a/output/archives/v1.3.0/velox-backend/troubleshooting/index.html
+++ b/output/archives/v1.3.0/velox-backend/troubleshooting/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
# velox backend only
rm -rf $SPARK_HOME/jars/flatbuffers-*
rm -rf $SPARK_HOME/jars/arrow-*
diff --git a/output/archives/v1.3.0/velox-backend/udf/index.html b/output/archives/v1.3.0/velox-backend/udf/index.html
index 39bc190..065bb72 100644
--- a/output/archives/v1.3.0/velox-backend/udf/index.html
+++ b/output/archives/v1.3.0/velox-backend/udf/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul:nth-of-type(2) > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > a, .site-nav > ul:nth-of-type(2) > li > ul > li > ul > li > ul > [...]
#include <velox/expression/VectorFunction.h>
#include <velox/udf/Udf.h>
diff --git a/output/assets/js/search-data.json b/output/assets/js/search-data.json
index c192687..b77e6fa 100644
--- a/output/assets/js/search-data.json
+++ b/output/assets/js/search-data.json
@@ -351,3252 +351,3259 @@
},"50": {
"doc": "Gluten References",
"title": "Gluten related Use Cases and Publications",
- "content": "For more information on Apache Gluten (incubating), including related articles and videos, please refer to the following links. | Title | Company | Publish Date | Reference | . | Gluten: A middle layer to offload Spark SQL to Native | Intel | Jul. 2022 | Link | . | Best Exploration of Columnar Shuffle Design | Intel | Jul. 2022 | Link | . | Gluten Introduction | Intel | Sept. 2022 | Link | . | Introducing Velox | Meta | Mar. 2023 | Link | . | Gluten in Kyligence(in Chines [...]
+ "content": "For more information on Apache Gluten (incubating), including related articles and videos, please refer to the following links. | Title | Company | Publish Date | Reference | . | Gluten: A middle layer to offload Spark SQL to Native | Intel | Jul. 2022 | Link | . | Best Exploration of Columnar Shuffle Design | Intel | Jul. 2022 | Link | . | Gluten Introduction | Intel | Sept. 2022 | Link | . | Introducing Velox | Meta | Mar. 2023 | Link | . | Gluten in Kyligence(in Chines [...]
"url": "/references/#gluten-related-use-cases-and-publications",
"relUrl": "/references/#gluten-related-use-cases-and-publications"
},"51": {
+ "doc": "Gluten References",
+ "title": "Related Documentation",
+ "content": "| Title | Project | Reference | . | Deploy Gluten with Apache Kyuubi | Apache Kyuubi | Link | . ",
+ "url": "/references/#related-documentation",
+
+ "relUrl": "/references/#related-documentation"
+ },"52": {
"doc": "Gluten References",
"title": "Gluten References",
"content": " ",
"url": "/references/",
"relUrl": "/references/"
- },"52": {
+ },"53": {
"doc": "Archives",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/#gluten-documents-by-version",
"relUrl": "/archives/#gluten-documents-by-version"
- },"53": {
+ },"54": {
"doc": "Archives",
"title": "Archives",
"content": " ",
"url": "/archives/",
"relUrl": "/archives/"
- },"54": {
+ },"55": {
"doc": "Documentation(Latest)",
"title": "Documentation(Latest)",
"content": " Click here to view the latest documentation . If you are not redirected, click here. ",
"url": "/docs/latest.html",
"relUrl": "/docs/latest.html"
- },"55": {
+ },"56": {
"doc": "v1.1.1/CPP Code Style",
"title": "Gluten CPP Core Guidelines",
"content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#gluten-cpp-core-guidelines",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#gluten-cpp-core-guidelines"
- },"56": {
+ },"57": {
"doc": "v1.1.1/CPP Code Style",
"title": "Philosophy",
"content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#philosophy",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#philosophy"
- },"57": {
+ },"58": {
"doc": "v1.1.1/CPP Code Style",
"title": "Code Formatting",
"content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#code-formatting",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#code-formatting"
- },"58": {
+ },"59": {
"doc": "v1.1.1/CPP Code Style",
"title": "Naming Conventions",
"content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#naming-conventions",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#naming-conventions"
- },"59": {
+ },"60": {
"doc": "v1.1.1/CPP Code Style",
"title": "Designs",
"content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#designs",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#designs"
- },"60": {
+ },"61": {
"doc": "v1.1.1/CPP Code Style",
"title": "Source File & Header File",
"content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#source-file--header-file",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#source-file--header-file"
- },"61": {
+ },"62": {
"doc": "v1.1.1/CPP Code Style",
"title": "Class",
"content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#class",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#class"
- },"62": {
+ },"63": {
"doc": "v1.1.1/CPP Code Style",
"title": "Function",
"content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#function",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#function"
- },"63": {
+ },"64": {
"doc": "v1.1.1/CPP Code Style",
"title": "Variable",
"content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#variable",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#variable"
- },"64": {
+ },"65": {
"doc": "v1.1.1/CPP Code Style",
"title": "Constant",
"content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#constant",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#constant"
- },"65": {
+ },"66": {
"doc": "v1.1.1/CPP Code Style",
"title": "Macro",
"content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#macro",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#macro"
- },"66": {
+ },"67": {
"doc": "v1.1.1/CPP Code Style",
"title": "Namespace",
"content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#namespace",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#namespace"
- },"67": {
+ },"68": {
"doc": "v1.1.1/CPP Code Style",
"title": "Resource Management",
"content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
"url": "/archives/v1.1.1/developers/cpp_code_style/#resource-management",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#resource-management"
- },"68": {
+ },"69": {
"doc": "v1.1.1/CPP Code Style",
"title": "Exception",
"content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#exception",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#exception"
- },"69": {
+ },"70": {
"doc": "v1.1.1/CPP Code Style",
"title": "Code Comment",
"content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#code-comment",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#code-comment"
- },"70": {
+ },"71": {
"doc": "v1.1.1/CPP Code Style",
"title": "References",
"content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
"url": "/archives/v1.1.1/developers/cpp_code_style/#references",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/#references"
- },"71": {
+ },"72": {
"doc": "v1.1.1/CPP Code Style",
"title": "v1.1.1/CPP Code Style",
"content": " ",
"url": "/archives/v1.1.1/developers/cpp_code_style/",
"relUrl": "/archives/v1.1.1/developers/cpp_code_style/"
- },"72": {
+ },"73": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to understand the key work of Gluten?",
"content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
"url": "/archives/v1.1.1/developers/howto/#how-to-understand-the-key-work-of-gluten",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-understand-the-key-work-of-gluten"
- },"73": {
+ },"74": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to debug in Gluten?",
"content": " ",
"url": "/archives/v1.1.1/developers/howto/#how-to-debug-in-gluten",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-debug-in-gluten"
- },"74": {
+ },"75": {
"doc": "v1.1.1/How To Use Gluten",
"title": "1 How to debug C++",
"content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
"url": "/archives/v1.1.1/developers/howto/#1-how-to-debug-c",
"relUrl": "/archives/v1.1.1/developers/howto/#1-how-to-debug-c"
- },"75": {
+ },"76": {
"doc": "v1.1.1/How To Use Gluten",
"title": "2 How to debug plan validation process",
"content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
"url": "/archives/v1.1.1/developers/howto/#2-how-to-debug-plan-validation-process",
"relUrl": "/archives/v1.1.1/developers/howto/#2-how-to-debug-plan-validation-process"
- },"76": {
+ },"77": {
"doc": "v1.1.1/How To Use Gluten",
"title": "3 How to debug Java/Scala",
"content": "wait to add . ",
"url": "/archives/v1.1.1/developers/howto/#3-how-to-debug-javascala",
"relUrl": "/archives/v1.1.1/developers/howto/#3-how-to-debug-javascala"
- },"77": {
+ },"78": {
"doc": "v1.1.1/How To Use Gluten",
"title": "4 How to debug with core-dump",
"content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
"url": "/archives/v1.1.1/developers/howto/#4-how-to-debug-with-core-dump",
"relUrl": "/archives/v1.1.1/developers/howto/#4-how-to-debug-with-core-dump"
- },"78": {
+ },"79": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to run TPC-H on Velox backend",
"content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox-be by workflow, you can refer to velox_be.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, prepare the d [...]
"url": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-h-on-velox-backend",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-h-on-velox-backend"
- },"79": {
+ },"80": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to run TPC-DS",
"content": "wait to add . ",
"url": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-ds",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-run-tpc-ds"
- },"80": {
+ },"81": {
"doc": "v1.1.1/How To Use Gluten",
"title": "How to track the memory exhaust problem",
"content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
"url": "/archives/v1.1.1/developers/howto/#how-to-track-the-memory-exhaust-problem",
"relUrl": "/archives/v1.1.1/developers/howto/#how-to-track-the-memory-exhaust-problem"
- },"81": {
+ },"82": {
"doc": "v1.1.1/How To Use Gluten",
"title": "v1.1.1/How To Use Gluten",
"content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
"url": "/archives/v1.1.1/developers/howto/",
"relUrl": "/archives/v1.1.1/developers/howto/"
- },"82": {
+ },"83": {
"doc": "v1.1.1/How To Release",
"title": "How to Release",
"content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/archives/v1.1.1/developers/howtorelease/#how-to-release",
"relUrl": "/archives/v1.1.1/developers/howtorelease/#how-to-release"
- },"83": {
+ },"84": {
"doc": "v1.1.1/How To Release",
"title": "Release Process",
"content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
"url": "/archives/v1.1.1/developers/howtorelease/#release-process",
"relUrl": "/archives/v1.1.1/developers/howtorelease/#release-process"
- },"84": {
+ },"85": {
"doc": "v1.1.1/How To Release",
"title": "v1.1.1/How To Release",
"content": " ",
"url": "/archives/v1.1.1/developers/howtorelease/",
"relUrl": "/archives/v1.1.1/developers/howtorelease/"
- },"85": {
+ },"86": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Generate Micro Benchmarks for Velox Backend",
"content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#generate-micro-benchmarks-for-velox-backend",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#generate-micro-benchmarks-for-velox-backend"
- },"86": {
+ },"87": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Try the example",
"content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#try-the-example",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#try-the-example"
- },"87": {
+ },"88": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Generate Substrait plan and input for any query",
"content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#generate-substrait-plan-and-input-for-any-query",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#generate-substrait-plan-and-input-for-any-query"
- },"88": {
+ },"89": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Save ouput to parquet to analyze",
"content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
"url": "/archives/v1.1.1/developers/microbenchmark/#save-ouput-to-parquet-to-analyze",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#save-ouput-to-parquet-to-analyze"
- },"89": {
+ },"90": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Add shuffle write process",
"content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#add-shuffle-write-process",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#add-shuffle-write-process"
- },"90": {
+ },"91": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "Simulate Spark with multiple processes and threads",
"content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
"url": "/archives/v1.1.1/developers/microbenchmark/#simulate-spark-with-multiple-processes-and-threads",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/#simulate-spark-with-multiple-processes-and-threads"
- },"91": {
+ },"92": {
"doc": "v1.1.1/Micro Benchmarks for Velox Backend",
"title": "v1.1.1/Micro Benchmarks for Velox Backend",
"content": " ",
"url": "/archives/v1.1.1/developers/microbenchmark/",
"relUrl": "/archives/v1.1.1/developers/microbenchmark/"
- },"92": {
+ },"93": {
"doc": "v1.1.1/New To Gluten",
"title": "Environment",
"content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
"url": "/archives/v1.1.1/developers/newto/#environment",
"relUrl": "/archives/v1.1.1/developers/newto/#environment"
- },"93": {
+ },"94": {
"doc": "v1.1.1/New To Gluten",
"title": "Openjdk8",
"content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
"url": "/archives/v1.1.1/developers/newto/#openjdk8",
"relUrl": "/archives/v1.1.1/developers/newto/#openjdk8"
- },"94": {
+ },"95": {
"doc": "v1.1.1/New To Gluten",
"title": "Maven 3.6.3 or above",
"content": "Maven Dowload Page And then set the environment setting. ",
"url": "/archives/v1.1.1/developers/newto/#maven-363-or-above",
"relUrl": "/archives/v1.1.1/developers/newto/#maven-363-or-above"
- },"95": {
+ },"96": {
"doc": "v1.1.1/New To Gluten",
"title": "GCC 9.4 or above",
"content": " ",
"url": "/archives/v1.1.1/developers/newto/#gcc-94-or-above",
"relUrl": "/archives/v1.1.1/developers/newto/#gcc-94-or-above"
- },"96": {
+ },"97": {
"doc": "v1.1.1/New To Gluten",
"title": "Compile gluten using debug mode",
"content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
"url": "/archives/v1.1.1/developers/newto/#compile-gluten-using-debug-mode",
"relUrl": "/archives/v1.1.1/developers/newto/#compile-gluten-using-debug-mode"
- },"97": {
+ },"98": {
"doc": "v1.1.1/New To Gluten",
"title": "Java/scala code development with Intellij",
"content": " ",
"url": "/archives/v1.1.1/developers/newto/#javascala-code-development-with-intellij",
"relUrl": "/archives/v1.1.1/developers/newto/#javascala-code-development-with-intellij"
- },"98": {
+ },"99": {
"doc": "v1.1.1/New To Gluten",
"title": "Linux intellij local debug",
"content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
"url": "/archives/v1.1.1/developers/newto/#linux-intellij-local-debug",
"relUrl": "/archives/v1.1.1/developers/newto/#linux-intellij-local-debug"
- },"99": {
+ },"100": {
"doc": "v1.1.1/New To Gluten",
"title": "Windows/Mac intellij remote debug",
"content": "If you have Ultimate intellij, you can try to debug remotely. ",
"url": "/archives/v1.1.1/developers/newto/#windowsmac-intellij-remote-debug",
"relUrl": "/archives/v1.1.1/developers/newto/#windowsmac-intellij-remote-debug"
- },"100": {
+ },"101": {
"doc": "v1.1.1/New To Gluten",
"title": "Set up gluten project",
"content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
"url": "/archives/v1.1.1/developers/newto/#set-up-gluten-project",
"relUrl": "/archives/v1.1.1/developers/newto/#set-up-gluten-project"
- },"101": {
+ },"102": {
"doc": "v1.1.1/New To Gluten",
"title": "Java/Scala code style",
"content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
"url": "/archives/v1.1.1/developers/newto/#javascala-code-style",
"relUrl": "/archives/v1.1.1/developers/newto/#javascala-code-style"
- },"102": {
+ },"103": {
"doc": "v1.1.1/New To Gluten",
"title": "CPP code development with Visual Studio Code",
"content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
"url": "/archives/v1.1.1/developers/newto/#cpp-code-development-with-visual-studio-code",
"relUrl": "/archives/v1.1.1/developers/newto/#cpp-code-development-with-visual-studio-code"
- },"103": {
+ },"104": {
"doc": "v1.1.1/New To Gluten",
"title": "Usage",
"content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
"url": "/archives/v1.1.1/developers/newto/#usage",
"relUrl": "/archives/v1.1.1/developers/newto/#usage"
- },"104": {
+ },"105": {
"doc": "v1.1.1/New To Gluten",
"title": "Usefule notes",
"content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
"url": "/archives/v1.1.1/developers/newto/#usefule-notes",
"relUrl": "/archives/v1.1.1/developers/newto/#usefule-notes"
- },"105": {
+ },"106": {
"doc": "v1.1.1/New To Gluten",
"title": "Debug cpp code with coredump",
"content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
"url": "/archives/v1.1.1/developers/newto/#debug-cpp-code-with-coredump",
"relUrl": "/archives/v1.1.1/developers/newto/#debug-cpp-code-with-coredump"
- },"106": {
+ },"107": {
"doc": "v1.1.1/New To Gluten",
"title": "Debug cpp with gdb",
"content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
"url": "/archives/v1.1.1/developers/newto/#debug-cpp-with-gdb",
"relUrl": "/archives/v1.1.1/developers/newto/#debug-cpp-with-gdb"
- },"107": {
+ },"108": {
"doc": "v1.1.1/New To Gluten",
"title": "Run TPC-H and TPC-DS",
"content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_be.yml . ",
"url": "/archives/v1.1.1/developers/newto/#run-tpc-h-and-tpc-ds",
"relUrl": "/archives/v1.1.1/developers/newto/#run-tpc-h-and-tpc-ds"
- },"108": {
+ },"109": {
"doc": "v1.1.1/New To Gluten",
"title": "Run gluten+velox on clean machine",
"content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
"url": "/archives/v1.1.1/developers/newto/#run-glutenvelox-on-clean-machine",
"relUrl": "/archives/v1.1.1/developers/newto/#run-glutenvelox-on-clean-machine"
- },"109": {
+ },"110": {
"doc": "v1.1.1/New To Gluten",
"title": "v1.1.1/New To Gluten",
"content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
"url": "/archives/v1.1.1/developers/newto/",
"relUrl": "/archives/v1.1.1/developers/newto/"
- },"110": {
+ },"111": {
"doc": "v1.1.1/Substrait Modifications",
"title": "Substrait Modifications in Gluten",
"content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
"url": "/archives/v1.1.1/developers/substrait/#substrait-modifications-in-gluten",
"relUrl": "/archives/v1.1.1/developers/substrait/#substrait-modifications-in-gluten"
- },"111": {
+ },"112": {
"doc": "v1.1.1/Substrait Modifications",
"title": "Modifications to algebra.proto",
"content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
"url": "/archives/v1.1.1/developers/substrait/#modifications-to-algebraproto",
"relUrl": "/archives/v1.1.1/developers/substrait/#modifications-to-algebraproto"
- },"112": {
+ },"113": {
"doc": "v1.1.1/Substrait Modifications",
"title": "Modifications to type.proto",
"content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
"url": "/archives/v1.1.1/developers/substrait/#modifications-to-typeproto",
"relUrl": "/archives/v1.1.1/developers/substrait/#modifications-to-typeproto"
- },"113": {
+ },"114": {
"doc": "v1.1.1/Substrait Modifications",
"title": "v1.1.1/Substrait Modifications",
"content": " ",
"url": "/archives/v1.1.1/developers/substrait/",
"relUrl": "/archives/v1.1.1/developers/substrait/"
- },"114": {
+ },"115": {
"doc": "v1.1.1/Docker script for CentOS 7",
"title": "v1.1.1/Docker script for CentOS 7",
"content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ sudo # gluten need maven version >=3.6.3 wget https://downloads.apache.org/m [...]
"url": "/archives/v1.1.1/developers/docker_centos7/",
"relUrl": "/archives/v1.1.1/developers/docker_centos7/"
- },"115": {
+ },"116": {
"doc": "v1.1.1/Docker script for CentOS 8",
"title": "v1.1.1/Docker script for CentOS 8",
"content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
"url": "/archives/v1.1.1/developers/docker_centos8",
"relUrl": "/archives/v1.1.1/developers/docker_centos8"
- },"116": {
+ },"117": {
"doc": "v1.1.1/Docker script for Ubuntu 22.04/20.04",
"title": "v1.1.1/Docker script for Ubuntu 22.04/20.04",
"content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
"url": "/archives/v1.1.1/developers/docker_ubuntu/",
"relUrl": "/archives/v1.1.1/developers/docker_ubuntu/"
- },"117": {
+ },"118": {
"doc": "v1.1.1/Developers",
"title": "Gluten Developers",
"content": "This document provides a developer overview of the project and covers the following topics: . ",
"url": "/archives/v1.1.1/developers/#gluten-developers",
"relUrl": "/archives/v1.1.1/developers/#gluten-developers"
- },"118": {
+ },"119": {
"doc": "v1.1.1/Developers",
"title": "v1.1.1/Developers",
"content": " ",
"url": "/archives/v1.1.1/developers/",
"relUrl": "/archives/v1.1.1/developers/"
- },"119": {
+ },"120": {
"doc": "v1.1.1/Velox Function Development",
"title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
"content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
"url": "/archives/v1.1.1/developers/velox_function_development/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
"relUrl": "/archives/v1.1.1/developers/velox_function_development/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"120": {
+ },"121": {
"doc": "v1.1.1/Velox Function Development",
"title": "v1.1.1/Velox Function Development",
"content": " ",
"url": "/archives/v1.1.1/developers/velox_function_development/",
"relUrl": "/archives/v1.1.1/developers/velox_function_development/"
- },"121": {
+ },"122": {
"doc": "v1.1.1/ClickHouse Backend",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.1.1/docs/clickhouse/#gluten-documents-by-version",
"relUrl": "/archives/v1.1.1/docs/clickhouse/#gluten-documents-by-version"
- },"122": {
+ },"123": {
"doc": "v1.1.1/ClickHouse Backend",
"title": "v1.1.1/ClickHouse Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/clickhouse/",
"relUrl": "/archives/v1.1.1/docs/clickhouse/"
- },"123": {
+ },"124": {
"doc": "v1.1.1/Getting Started with ClickHouse Backend",
"title": "ClickHouse Backend",
"content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
"url": "/archives/v1.1.1/docs/clickhouse/getting-started/#clickhouse-backend",
"relUrl": "/archives/v1.1.1/docs/clickhouse/getting-started/#clickhouse-backend"
- },"124": {
+ },"125": {
"doc": "v1.1.1/Getting Started with ClickHouse Backend",
"title": "v1.1.1/Getting Started with ClickHouse Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/clickhouse/getting-started/",
"relUrl": "/archives/v1.1.1/docs/clickhouse/getting-started/"
- },"125": {
+ },"126": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Supported Version",
"content": "| Type | Version |——-|——————————| Spark | 3.2.2, 3.3.1 | OS | Ubuntu20.04/22.04, Centos7/8 | jdk | openjdk8 | scala | 2.12 . Spark3.4.0 support is still WIP. TPCH/DS can pass, UT is not yet passed. There are pending PRs for jdk11 support. Currently, the mvn script can automatically fetch and build all dependency libraries incluing Velox. Our nightly build still use Velox under oap-project. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#supported-version",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#supported-version"
- },"126": {
+ },"127": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Prerequisite",
"content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Gluten builds with Spark3.2.x and Spark3.3.x now but only fully tested in CI with 3.2.2 and 3.3.1. We will add/update supported/tested versions according to the upstream changes. we need to set up the JAVA_HOME env. Currently, java 8 is required and the support for j [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#prerequisite",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#prerequisite"
- },"127": {
+ },"128": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Build Gluten with Velox Backend",
"content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . cd /path/to/gluten ## The script builds two jars for spark 3.2.2 and 3.3.1./dev/buildbundle-veloxbe.sh ## After a complete build, if you need to re-build the project and only some gluten code is changed, ## you can use the following command to skip building velox and protobuf. [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-velox-backend",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-velox-backend"
- },"128": {
+ },"129": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Dependency library deployment",
"content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#dependency-library-deployment",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#dependency-library-deployment"
- },"129": {
+ },"130": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "HDFS support",
"content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#hdfs-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#hdfs-support"
- },"130": {
+ },"131": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "AWS S3 support",
"content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#aws-s3-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#aws-s3-support"
- },"131": {
+ },"132": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Celeborn support",
"content": "Gluten with velox backend supports Celeborn as remote shuffle service. Below introduction is used to enable this feature . First refer to this URL(https://github.com/apache/incubator-celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable rss profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Prss -DskipTests . Then add the Gluten and Spark Celeborn Client packages to your Spark application’s classpath(usua [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#celeborn-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#celeborn-support"
- },"132": {
+ },"133": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "DeltaLake Support",
"content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with velox backends [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#deltalake-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#deltalake-support"
- },"133": {
+ },"134": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Iceberg Support",
"content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Then, put the additional gluten-iceberg jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-iceberg jar is in gluten-iceberg/target directory. After the two steps, you can query icebe [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#iceberg-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#iceberg-support"
- },"134": {
+ },"135": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Coverage",
"content": "Spark3.3 has 387 functions in total. ~240 are commonly used. Velox’s functions have two category, Presto and Spark. Presto has 124 functions implemented. Spark has 62 functions. Spark functions are verified to have the same result as Vanilla Spark. Some Presto functions have the same result as Vanilla Spark but some others have different. Gluten prefer to use Spark functions firstly. If it’s not in Spark’s list but implemented in Presto, we currently offload to Presto one [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#coverage",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#coverage"
- },"135": {
+ },"136": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Spill (Experimental)",
"content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#spill-experimental",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#spill-experimental"
- },"136": {
+ },"137": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Velox User-Defined Functions (UDF)",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#velox-user-defined-functions-udf",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#velox-user-defined-functions-udf"
- },"137": {
+ },"138": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Introduction",
"content": "Velox backend supports User-Defined Functions (UDF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#introduction",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#introduction"
- },"138": {
+ },"139": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Creating a UDF library",
"content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#creating-a-udf-library",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#creating-a-udf-library"
- },"139": {
+ },"140": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Building the UDF library",
"content": "To build the UDF library, users need to compile the C++ code and link to libvelox.so. It’s recommended to create a CMakeLists.txt for the project. Here’s an example: . project(myudf) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(GLUTEN_HOME /path/to/gluten) add_library(myudf SHARED \"MyUDF.cpp\") find_library(VELOX_LIBRARY REQUIRED NAMES velox HINTS ${GLUTEN_HOME}/cpp/build/releases NO_DEFAULT_PATH) target_include_directories(myudf PRIVATE ${GLUTEN_HO [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#building-the-udf-library",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#building-the-udf-library"
- },"140": {
+ },"141": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Using UDF in Gluten",
"content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the libray paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#using-udf-in-gluten",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#using-udf-in-gluten"
- },"141": {
+ },"142": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Try the example",
"content": "We provided an Velox UDF example file MyUDF.cpp. After building gluten cpp, you can find the example library at /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so . Start spark-shell or spark-sql with below configuration . --files /path/to/gluten/cpp/build/velox/udf/examples/libmyudf.so --conf spark.gluten.sql.columnar.backend.velox.udfLibraryPaths=libmyudf.so . Run query. The functions myudf1 and myudf2 increment the input value by a constant of 5 . select myudf1(1 [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#try-the-example",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#try-the-example"
- },"142": {
+ },"143": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "High-Bandwidth Memory (HBM) support",
"content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#high-bandwidth-memory-hbm-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#high-bandwidth-memory-hbm-support"
- },"143": {
+ },"144": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Build Gluten with HBM",
"content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds two jars for spark 3.2.2 and 3.3.1./dev/buildbundle-veloxbe.sh --enable_hbm=ON . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-hbm",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-hbm"
- },"144": {
+ },"145": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Configure and enable HBM in Spark Application",
"content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#configure-and-enable-hbm-in-spark-application",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#configure-and-enable-hbm-in-spark-application"
- },"145": {
+ },"146": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Intel® QuickAssist Technology (QAT) support",
"content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#intel-quickassist-technology-qat-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#intel-quickassist-technology-qat-support"
- },"146": {
+ },"147": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Software Requirements",
"content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#software-requirements",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#software-requirements"
- },"147": {
+ },"148": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Build Gluten with QAT",
"content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-qat",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-qat"
- },"148": {
+ },"149": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Enable QAT with Gzip/Zstd for shuffle compression",
"content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#enable-qat-with-gzipzstd-for-shuffle-compression",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#enable-qat-with-gzipzstd-for-shuffle-compression"
- },"149": {
+ },"150": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "QAT driver references",
"content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#qat-driver-references",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#qat-driver-references"
- },"150": {
+ },"151": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
"content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#intel-in-memory-analytics-accelerator-iaaiax-support",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#intel-in-memory-analytics-accelerator-iaaiax-support"
- },"151": {
+ },"152": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Build Gluten with IAA",
"content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-iaa",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#build-gluten-with-iaa"
- },"152": {
+ },"153": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Enable IAA with Gzip Compression for shuffle compression",
"content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#enable-iaa-with-gzip-compression-for-shuffle-compression",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#enable-iaa-with-gzip-compression-for-shuffle-compression"
- },"153": {
+ },"154": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "IAA references",
"content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#iaa-references",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#iaa-references"
- },"154": {
+ },"155": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
"content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
- },"155": {
+ },"156": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Data preparation",
"content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#data-preparation",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#data-preparation"
- },"156": {
+ },"157": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Submit the Spark SQL job",
"content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#submit-the-spark-sql-job",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#submit-the-spark-sql-job"
- },"157": {
+ },"158": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Result",
"content": "wholestagetransformer indicates that the offload works. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#result",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#result"
- },"158": {
+ },"159": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Performance",
"content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#performance",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#performance"
- },"159": {
+ },"160": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "External reference setup",
"content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#external-reference-setup",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#external-reference-setup"
- },"160": {
+ },"161": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Gluten UI",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-ui",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-ui"
- },"161": {
+ },"162": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Gluten event",
"content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-event",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-event"
- },"162": {
+ },"163": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "SQL tab",
"content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#sql-tab",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#sql-tab"
- },"163": {
+ },"164": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "History server",
"content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#history-server",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#history-server"
- },"164": {
+ },"165": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Native plan string",
"content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-string",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-string"
- },"165": {
+ },"166": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Native plan with stats",
"content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
"url": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-with-stats",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#native-plan-with-stats"
- },"166": {
+ },"167": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "Gluten Implicits",
"content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
"url": "/archives/v1.1.1/docs/velox/getting-started/#gluten-implicits",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/#gluten-implicits"
- },"167": {
+ },"168": {
"doc": "v1.1.1/Getting-Started with Velox Backend",
"title": "v1.1.1/Getting-Started with Velox Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/getting-started/",
"relUrl": "/archives/v1.1.1/docs/velox/getting-started/"
- },"168": {
+ },"169": {
"doc": "v1.1.1/Velox Backend",
"title": "Gluten Velox Backend Documents",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/#gluten-velox-backend-documents",
"relUrl": "/archives/v1.1.1/docs/velox/#gluten-velox-backend-documents"
- },"169": {
+ },"170": {
"doc": "v1.1.1/Velox Backend",
"title": "v1.1.1/Velox Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/",
"relUrl": "/archives/v1.1.1/docs/velox/"
- },"170": {
+ },"171": {
"doc": "v1.1.1/Build Parameters for Velox Backend",
"title": "Build Parameters",
"content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | ON | . | build_protob [...]
"url": "/archives/v1.1.1/docs/velox/build_parameters/#build-parameters",
"relUrl": "/archives/v1.1.1/docs/velox/build_parameters/#build-parameters"
- },"171": {
+ },"172": {
"doc": "v1.1.1/Build Parameters for Velox Backend",
"title": "Gluten Jar for Deployment",
"content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . ",
"url": "/archives/v1.1.1/docs/velox/build_parameters/#gluten-jar-for-deployment",
"relUrl": "/archives/v1.1.1/docs/velox/build_parameters/#gluten-jar-for-deployment"
- },"172": {
+ },"173": {
"doc": "v1.1.1/Build Parameters for Velox Backend",
"title": "v1.1.1/Build Parameters for Velox Backend",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/build_parameters/",
"relUrl": "/archives/v1.1.1/docs/velox/build_parameters/"
- },"173": {
+ },"174": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Working with GCS",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/gcs/#working-with-gcs",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#working-with-gcs"
- },"174": {
+ },"175": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Installing the gcloud CLI",
"content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
"url": "/archives/v1.1.1/docs/velox/gcs/#installing-the-gcloud-cli",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#installing-the-gcloud-cli"
- },"175": {
+ },"176": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Configuring GCS using a user account",
"content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
"url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-user-account",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-user-account"
- },"176": {
+ },"177": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Configuring GCS using a credential file",
"content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
"url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-credential-file",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-using-a-credential-file"
- },"177": {
+ },"178": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "Configuring GCS endpoints",
"content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
"url": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-endpoints",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/#configuring-gcs-endpoints"
- },"178": {
+ },"179": {
"doc": "v1.1.1/Using GCS with Gluten",
"title": "v1.1.1/Using GCS with Gluten",
"content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.1.1/docs/velox/gcs/",
"relUrl": "/archives/v1.1.1/docs/velox/gcs/"
- },"179": {
+ },"180": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Working with S3",
"content": " ",
"url": "/archives/v1.1.1/docs/velox/s3/#working-with-s3",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#working-with-s3"
- },"180": {
+ },"181": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Configuring S3 endpoint",
"content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
"url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-endpoint",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-endpoint"
- },"181": {
+ },"182": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Configuring S3 instance credentials",
"content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
"url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-instance-credentials",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-instance-credentials"
- },"182": {
+ },"183": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Configuring S3 IAM roles",
"content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
"url": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-iam-roles",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#configuring-s3-iam-roles"
- },"183": {
+ },"184": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Other authentatication methods are not supported yet",
"content": ". | AWS temporary credential | . ",
"url": "/archives/v1.1.1/docs/velox/s3/#other-authentatication-methods-are-not-supported-yet",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#other-authentatication-methods-are-not-supported-yet"
- },"184": {
+ },"185": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Log granularity of AWS C++ SDK in velox",
"content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
"url": "/archives/v1.1.1/docs/velox/s3/#log-granularity-of-aws-c-sdk-in-velox",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#log-granularity-of-aws-c-sdk-in-velox"
- },"185": {
+ },"186": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from HDFS/S3. The feature is very useful if remote storage is slow, e.g., reading from a public S3 bucket and stronger performance is desired. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage, and the future reading requests on already cached blocks will be serviced from local cache files. To enable the local caching feature, below configurations are required: . spark.g [...]
"url": "/archives/v1.1.1/docs/velox/s3/#local-caching-support",
"relUrl": "/archives/v1.1.1/docs/velox/s3/#local-caching-support"
- },"186": {
+ },"187": {
"doc": "v1.1.1/Using S3 with Gluten",
"title": "v1.1.1/Using S3 with Gluten",
"content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.1.1/docs/velox/s3/",
"relUrl": "/archives/v1.1.1/docs/velox/s3/"
- },"187": {
+ },"188": {
"doc": "v1.1.1/Documentation",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.1.1/docs/#gluten-documents-by-version",
"relUrl": "/archives/v1.1.1/docs/#gluten-documents-by-version"
- },"188": {
+ },"189": {
"doc": "v1.1.1/Documentation",
"title": "v1.1.1/Documentation",
"content": " ",
"url": "/archives/v1.1.1/docs/",
"relUrl": "/archives/v1.1.1/docs/"
- },"189": {
+ },"190": {
"doc": "v1.1.1",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.1.1/#gluten-documents-by-version",
"relUrl": "/archives/v1.1.1/#gluten-documents-by-version"
- },"190": {
+ },"191": {
"doc": "v1.1.1",
"title": "v1.1.1",
"content": " ",
"url": "/archives/v1.1.1/",
"relUrl": "/archives/v1.1.1/"
- },"191": {
+ },"192": {
"doc": "Configuration(v1.2.1)",
"title": "Spark Configurations for Gluten Plugin",
"content": "There are many configurations could impact the Gluten Plugin performance and can be fine-tuned in Spark. You can add these configurations into spark-defaults.conf to enable or disable the setting. ",
"url": "/archives/v1.2.1/docs/configuration/#spark-configurations-for-gluten-plugin",
"relUrl": "/archives/v1.2.1/docs/configuration/#spark-configurations-for-gluten-plugin"
- },"192": {
+ },"193": {
"doc": "Configuration(v1.2.1)",
"title": "Spark parameters",
"content": "| Parameters | Description | Recommend Setting | . | spark.driver.extraClassPath | To add Gluten Plugin jar file in Spark Driver | /path/to/jar_file | . | spark.executor.extraClassPath | To add Gluten Plugin jar file in Spark Executor | /path/to/jar_file | . | spark.executor.memory | To set up how much memory to be used for Spark Executor. | | . | spark.memory.offHeap.size | To set up how much memory to be used for Java OffHeap. Please notice Gluten Plugin will leverage [...]
"url": "/archives/v1.2.1/docs/configuration/#spark-parameters",
"relUrl": "/archives/v1.2.1/docs/configuration/#spark-parameters"
- },"193": {
+ },"194": {
"doc": "Configuration(v1.2.1)",
"title": "Velox Parameters",
"content": "The following configurations are related to Velox settings. | Parameters | Description | Recommend Setting | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The m [...]
"url": "/archives/v1.2.1/docs/configuration/#velox-parameters",
"relUrl": "/archives/v1.2.1/docs/configuration/#velox-parameters"
- },"194": {
+ },"195": {
"doc": "Configuration(v1.2.1)",
"title": "Configuration(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/configuration/",
"relUrl": "/archives/v1.2.1/docs/configuration/"
- },"195": {
+ },"196": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Gluten CPP Core Guidelines",
"content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#gluten-cpp-core-guidelines"
- },"196": {
+ },"197": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Philosophy",
"content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#philosophy",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#philosophy"
- },"197": {
+ },"198": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Code Formatting",
"content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-formatting",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-formatting"
- },"198": {
+ },"199": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Naming Conventions",
"content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#naming-conventions",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#naming-conventions"
- },"199": {
+ },"200": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Designs",
"content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#designs",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#designs"
- },"200": {
+ },"201": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Source File & Header File",
"content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#source-file--header-file",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#source-file--header-file"
- },"201": {
+ },"202": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Class",
"content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#class",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#class"
- },"202": {
+ },"203": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Function",
"content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#function",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#function"
- },"203": {
+ },"204": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Variable",
"content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#variable",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#variable"
- },"204": {
+ },"205": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Constant",
"content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#constant",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#constant"
- },"205": {
+ },"206": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Macro",
"content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#macro",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#macro"
- },"206": {
+ },"207": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Namespace",
"content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#namespace",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#namespace"
- },"207": {
+ },"208": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Resource Management",
"content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#resource-management",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#resource-management"
- },"208": {
+ },"209": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Exception",
"content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#exception",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#exception"
- },"209": {
+ },"210": {
"doc": "CPP Code Style(v1.2.1)",
"title": "Code Comment",
"content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-comment",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#code-comment"
- },"210": {
+ },"211": {
"doc": "CPP Code Style(v1.2.1)",
"title": "References",
"content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/#references",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/#references"
- },"211": {
+ },"212": {
"doc": "CPP Code Style(v1.2.1)",
"title": "CPP Code Style(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/cpp-code-style/",
"relUrl": "/archives/v1.2.1/docs/developers/cpp-code-style/"
- },"212": {
+ },"213": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to understand the key work of Gluten?",
"content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-understand-the-key-work-of-gluten"
- },"213": {
+ },"214": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to debug in Gluten?",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-debug-in-gluten",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-debug-in-gluten"
- },"214": {
+ },"215": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "1 How to debug C++",
"content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#1-how-to-debug-c",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#1-how-to-debug-c"
- },"215": {
+ },"216": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "2 How to debug plan validation process",
"content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#2-how-to-debug-plan-validation-process"
- },"216": {
+ },"217": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "3 How to debug Java/Scala",
"content": "wait to add . ",
"url": "/archives/v1.2.1/docs/developers/how-to-use/#3-how-to-debug-javascala",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#3-how-to-debug-javascala"
- },"217": {
+ },"218": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "4 How to debug with core-dump",
"content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
"url": "/archives/v1.2.1/docs/developers/how-to-use/#4-how-to-debug-with-core-dump",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#4-how-to-debug-with-core-dump"
- },"218": {
+ },"219": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to run TPC-H on Velox backend",
"content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox with docker by workflow, you can refer to velox_docker.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend"
- },"219": {
+ },"220": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to run TPC-DS",
"content": "wait to add . ",
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-ds",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-run-tpc-ds"
- },"220": {
+ },"221": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How to track the memory exhaust problem",
"content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/#how-to-track-the-memory-exhaust-problem"
- },"221": {
+ },"222": {
"doc": "How To Use Gluten(v1.2.1)",
"title": "How To Use Gluten(v1.2.1)",
"content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
"url": "/archives/v1.2.1/docs/developers/how-to-use/",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-use/"
- },"222": {
+ },"223": {
"doc": "How To Release(v1.2.1)",
"title": "How to Release",
"content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/archives/v1.2.1/docs/developers/how-to-release/#how-to-release",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-release/#how-to-release"
- },"223": {
+ },"224": {
"doc": "How To Release(v1.2.1)",
"title": "Release Process",
"content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
"url": "/archives/v1.2.1/docs/developers/how-to-release/#release-process",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-release/#release-process"
- },"224": {
+ },"225": {
"doc": "How To Release(v1.2.1)",
"title": "How To Release(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/how-to-release/",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-release/"
- },"225": {
+ },"226": {
"doc": "How To Scan Security issues(v1.2.1)",
"title": "How to scan the security issues",
"content": "This section outlines the steps to use tools to scan Apache Gluten (incubating) source code and make sure no vulnerability issues in the code. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/archives/v1.2.1/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/#how-to-scan-the-security-issues"
- },"226": {
+ },"227": {
"doc": "How To Scan Security issues(v1.2.1)",
"title": "Scan Security Process",
"content": "Before every Apache Gluten (incubating) release, we need to ensure there is no vulnerability issue in the source code. We use Trivy as the tool to scan all the security issues. | Install Trivy, please follow the steps to install Trivy: Trivy Installation . | Configuring Trivy, please follow the guide to configure Trivy for specific operation: Trivy Configuration . | Run Trivy File System Scan with the source code. Below is an example about how we run Trivy scan with Apach [...]
"url": "/archives/v1.2.1/docs/developers/how-to-security-scan/#scan-security-process",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/#scan-security-process"
- },"227": {
+ },"228": {
"doc": "How To Scan Security issues(v1.2.1)",
"title": "How To Scan Security issues(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/how-to-security-scan/",
"relUrl": "/archives/v1.2.1/docs/developers/how-to-security-scan/"
- },"228": {
+ },"229": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Generate Micro Benchmarks for Velox Backend",
"content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend"
- },"229": {
+ },"230": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Try the example",
"content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#try-the-example",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#try-the-example"
- },"230": {
+ },"231": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Generate Substrait plan and input for any query",
"content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query"
- },"231": {
+ },"232": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Save ouput to parquet to analyze",
"content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze"
- },"232": {
+ },"233": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Add shuffle write process",
"content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#add-shuffle-write-process",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#add-shuffle-write-process"
- },"233": {
+ },"234": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Simulate Spark with multiple processes and threads",
"content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads"
- },"234": {
+ },"235": {
"doc": "Micro Benchmarks for Velox Backend(v1.2.1)",
"title": "Micro Benchmarks for Velox Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/microbenchmarks/",
"relUrl": "/archives/v1.2.1/docs/developers/microbenchmarks/"
- },"235": {
+ },"236": {
"doc": "New To Gluten(v1.2.1)",
"title": "Environment",
"content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
"url": "/archives/v1.2.1/docs/developers/new-to/#environment",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#environment"
- },"236": {
+ },"237": {
"doc": "New To Gluten(v1.2.1)",
"title": "Openjdk8",
"content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#openjdk8",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#openjdk8"
- },"237": {
+ },"238": {
"doc": "New To Gluten(v1.2.1)",
"title": "Maven 3.6.3 or above",
"content": "Maven Dowload Page And then set the environment setting. ",
"url": "/archives/v1.2.1/docs/developers/new-to/#maven-363-or-above",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#maven-363-or-above"
- },"238": {
+ },"239": {
"doc": "New To Gluten(v1.2.1)",
"title": "GCC 9.4 or above",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/new-to/#gcc-94-or-above",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#gcc-94-or-above"
- },"239": {
+ },"240": {
"doc": "New To Gluten(v1.2.1)",
"title": "Compile gluten using debug mode",
"content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#compile-gluten-using-debug-mode",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#compile-gluten-using-debug-mode"
- },"240": {
+ },"241": {
"doc": "New To Gluten(v1.2.1)",
"title": "Java/scala code development with Intellij",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-development-with-intellij",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-development-with-intellij"
- },"241": {
+ },"242": {
"doc": "New To Gluten(v1.2.1)",
"title": "Linux intellij local debug",
"content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#linux-intellij-local-debug",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#linux-intellij-local-debug"
- },"242": {
+ },"243": {
"doc": "New To Gluten(v1.2.1)",
"title": "Windows/Mac intellij remote debug",
"content": "If you have Ultimate intellij, you can try to debug remotely. ",
"url": "/archives/v1.2.1/docs/developers/new-to/#windowsmac-intellij-remote-debug",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#windowsmac-intellij-remote-debug"
- },"243": {
+ },"244": {
"doc": "New To Gluten(v1.2.1)",
"title": "Set up gluten project",
"content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
"url": "/archives/v1.2.1/docs/developers/new-to/#set-up-gluten-project",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#set-up-gluten-project"
- },"244": {
+ },"245": {
"doc": "New To Gluten(v1.2.1)",
"title": "Java/Scala code style",
"content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-style",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#javascala-code-style"
- },"245": {
+ },"246": {
"doc": "New To Gluten(v1.2.1)",
"title": "CPP code development with Visual Studio Code",
"content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#cpp-code-development-with-visual-studio-code",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#cpp-code-development-with-visual-studio-code"
- },"246": {
+ },"247": {
"doc": "New To Gluten(v1.2.1)",
"title": "Usage",
"content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#usage",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#usage"
- },"247": {
+ },"248": {
"doc": "New To Gluten(v1.2.1)",
"title": "Usefule notes",
"content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#usefule-notes",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#usefule-notes"
- },"248": {
+ },"249": {
"doc": "New To Gluten(v1.2.1)",
"title": "Debug cpp code with coredump",
"content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-code-with-coredump",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-code-with-coredump"
- },"249": {
+ },"250": {
"doc": "New To Gluten(v1.2.1)",
"title": "Debug cpp with gdb",
"content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-with-gdb",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#debug-cpp-with-gdb"
- },"250": {
+ },"251": {
"doc": "New To Gluten(v1.2.1)",
"title": "Run TPC-H and TPC-DS",
"content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_docker.yml . ",
"url": "/archives/v1.2.1/docs/developers/new-to/#run-tpc-h-and-tpc-ds",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#run-tpc-h-and-tpc-ds"
- },"251": {
+ },"252": {
"doc": "New To Gluten(v1.2.1)",
"title": "Run gluten+velox on clean machine",
"content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
"url": "/archives/v1.2.1/docs/developers/new-to/#run-glutenvelox-on-clean-machine",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/#run-glutenvelox-on-clean-machine"
- },"252": {
+ },"253": {
"doc": "New To Gluten(v1.2.1)",
"title": "New To Gluten(v1.2.1)",
"content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
"url": "/archives/v1.2.1/docs/developers/new-to/",
"relUrl": "/archives/v1.2.1/docs/developers/new-to/"
- },"253": {
+ },"254": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Substrait Modifications in Gluten",
"content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
"url": "/archives/v1.2.1/docs/developers/substrait/#substrait-modifications-in-gluten",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/#substrait-modifications-in-gluten"
- },"254": {
+ },"255": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Modifications to algebra.proto",
"content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
"url": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-algebraproto",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-algebraproto"
- },"255": {
+ },"256": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Modifications to type.proto",
"content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
"url": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-typeproto",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/#modifications-to-typeproto"
- },"256": {
+ },"257": {
"doc": "Substrait Modifications(v1.2.1)",
"title": "Substrait Modifications(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/substrait/",
"relUrl": "/archives/v1.2.1/docs/developers/substrait/"
- },"257": {
+ },"258": {
"doc": "Docker script for CentOS 7(v1.2.1)",
"title": "Docker script for CentOS 7(v1.2.1)",
"content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git236 \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ autoconf \\ autoconf-archive \\ automake \\ perl-IPC-Cmd \\ libicu-devel \\ [...]
"url": "/archives/v1.2.1/docs/developers/docker-centos7/",
"relUrl": "/archives/v1.2.1/docs/developers/docker-centos7/"
- },"258": {
+ },"259": {
"doc": "Docker script for CentOS 8(v1.2.1)",
"title": "Docker script for CentOS 8(v1.2.1)",
"content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
"url": "/archives/v1.2.1/docs/developers/docker-centos8/",
"relUrl": "/archives/v1.2.1/docs/developers/docker-centos8/"
- },"259": {
+ },"260": {
"doc": "Docker script for Ubuntu 22.04/20.04(v1.2.1)",
"title": "Docker script for Ubuntu 22.04/20.04(v1.2.1)",
"content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
"url": "/archives/v1.2.1/docs/developers/docker-ubuntu/",
"relUrl": "/archives/v1.2.1/docs/developers/docker-ubuntu/"
- },"260": {
+ },"261": {
"doc": "Developers(v1.2.1)",
"title": "Gluten Developers",
"content": "This document provides a developer overview of the project and covers the following topics: . ",
"url": "/archives/v1.2.1/docs/developers/#gluten-developers",
"relUrl": "/archives/v1.2.1/docs/developers/#gluten-developers"
- },"261": {
+ },"262": {
"doc": "Developers(v1.2.1)",
"title": "Developers(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/",
"relUrl": "/archives/v1.2.1/docs/developers/"
- },"262": {
+ },"263": {
"doc": "Velox Function Development(v1.2.1)",
"title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
"content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
"url": "/archives/v1.2.1/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
"relUrl": "/archives/v1.2.1/docs/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"263": {
+ },"264": {
"doc": "Velox Function Development(v1.2.1)",
"title": "Velox Function Development(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/developers/velox-function-dev/",
"relUrl": "/archives/v1.2.1/docs/developers/velox-function-dev/"
- },"264": {
+ },"265": {
"doc": "Getting start with ClickHouse Backend(v1.2.1)",
"title": "ClickHouse Backend",
"content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
"url": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/#clickhouse-backend",
"relUrl": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/#clickhouse-backend"
- },"265": {
+ },"266": {
"doc": "Getting start with ClickHouse Backend(v1.2.1)",
"title": "Getting start with ClickHouse Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/",
"relUrl": "/archives/v1.2.1/docs/getting-started/clickhouse-backend/"
- },"266": {
+ },"267": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Supported Version",
"content": "| Type | Version | . | Spark | 3.2.2, 3.3.1, 3.4.2, 3.5.1 | . | OS | Ubuntu20.04/22.04, Centos7/8 | . | jdk | openjdk8/jdk17 | . | scala | 2.12 | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#supported-version",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#supported-version"
- },"267": {
+ },"268": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Prerequisite",
"content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos7/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.2 and 3.5.1. We need to set up the JAVA_HOME env. Currently, Gluten supports java 8 and java 17. For x86_64 . ## make sure jdk8 is used export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#prerequisite",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#prerequisite"
- },"268": {
+ },"269": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with Velox Backend",
"content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . First time build for all supported spark versions./dev/buildbundle-veloxbe.sh . After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and setting up build dependencies./dev/buildbundle-veloxbe.sh --enable_ep [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-velox-backend",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-velox-backend"
- },"269": {
+ },"270": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Dependency library deployment",
"content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#dependency-library-deployment",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#dependency-library-deployment"
- },"270": {
+ },"271": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "HDFS support",
"content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#hdfs-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#hdfs-support"
- },"271": {
+ },"272": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Azure Blob File System (ABFS) support",
"content": "Velox supports ABFS with the open source Azure SDK for C++ and Gluten uses the Velox ABFS connector to connect with ABFS. The build option for ABFS (enable_abfs) must be set to enable this feature as listed below. cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_abfs=ON . Please refer Velox ABFS part for more detailed configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#azure-blob-file-system-abfs-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#azure-blob-file-system-abfs-support"
- },"272": {
+ },"273": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "AWS S3 support",
"content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#aws-s3-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#aws-s3-support"
- },"273": {
+ },"274": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Celeborn support",
"content": "Gluten with velox backend supports Celeborn as remote shuffle service. Currently, the supported Celeborn versions are 0.3.x, 0.4.x and 0.5.0. Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable celeborn profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests . Then add the Gluten and Spa [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#celeborn-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#celeborn-support"
- },"274": {
+ },"275": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Uniffle support",
"content": "Uniffle with velox backend supports Uniffle as remote shuffle service. Currently, the supported Uniffle versions are 0.8.0. First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with uniffle. When compiling the Gluten Java module, it’s required to enable uniffle profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests . Then add the Uniffle and Spark Celeborn Client packages to your Spark application’s classpath(usu [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#uniffle-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#uniffle-support"
- },"275": {
+ },"276": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "DeltaLake Support",
"content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta-XX-SNAPSHOT.jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with vel [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#deltalake-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#deltalake-support"
- },"276": {
+ },"277": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Iceberg Support",
"content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan’s fallback. After the two steps, you can query [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#iceberg-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#iceberg-support"
- },"277": {
+ },"278": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Coverage",
"content": "Spark3.3 has 387 functions in total. ~240 are commonly used. To get the support status of all Spark built-in functions, please refer to Velox Backend’s Supported Operators & Functions. Velox doesn’t support ANSI mode), so as Gluten. Once ANSI mode is enabled in Spark config, Gluten will fallback to Vanilla Spark. To identify what can be offloaded in a query and detailed fallback reasons, user can follow below steps to retrieve corresponding logs. 1) Enable Gluten by p [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#coverage",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#coverage"
- },"278": {
+ },"279": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Spill (Experimental)",
"content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#spill-experimental",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#spill-experimental"
- },"279": {
+ },"280": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": "Please check the VeloxNativeUDF.md for more detailed usage and configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"280": {
+ },"281": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "High-Bandwidth Memory (HBM) support",
"content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#high-bandwidth-memory-hbm-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#high-bandwidth-memory-hbm-support"
- },"281": {
+ },"282": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with HBM",
"content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.2 and 3.5.1./dev/buildbundle-veloxbe.sh -- [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-hbm",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-hbm"
- },"282": {
+ },"283": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Configure and enable HBM in Spark Application",
"content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application"
- },"283": {
+ },"284": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Intel® QuickAssist Technology (QAT) support",
"content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-quickassist-technology-qat-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-quickassist-technology-qat-support"
- },"284": {
+ },"285": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Software Requirements",
"content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#software-requirements",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#software-requirements"
- },"285": {
+ },"286": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with QAT",
"content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-qat",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-qat"
- },"286": {
+ },"287": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Enable QAT with Gzip/Zstd for shuffle compression",
"content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression"
- },"287": {
+ },"288": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "QAT driver references",
"content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#qat-driver-references",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#qat-driver-references"
- },"288": {
+ },"289": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
"content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support"
- },"289": {
+ },"290": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Build Gluten with IAA",
"content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-iaa",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#build-gluten-with-iaa"
- },"290": {
+ },"291": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Enable IAA with Gzip Compression for shuffle compression",
"content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression"
- },"291": {
+ },"292": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "IAA references",
"content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#iaa-references",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#iaa-references"
- },"292": {
+ },"293": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
"content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
- },"293": {
+ },"294": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Data preparation",
"content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#data-preparation",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#data-preparation"
- },"294": {
+ },"295": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Submit the Spark SQL job",
"content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#submit-the-spark-sql-job",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#submit-the-spark-sql-job"
- },"295": {
+ },"296": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Result",
"content": "wholestagetransformer indicates that the offloading works. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#result",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#result"
- },"296": {
+ },"297": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Performance",
"content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#performance",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#performance"
- },"297": {
+ },"298": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "External reference setup",
"content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#external-reference-setup",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#external-reference-setup"
- },"298": {
+ },"299": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Gluten UI",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-ui",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-ui"
- },"299": {
+ },"300": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Gluten event",
"content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-event",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-event"
- },"300": {
+ },"301": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "SQL tab",
"content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#sql-tab",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#sql-tab"
- },"301": {
+ },"302": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "History server",
"content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#history-server",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#history-server"
- },"302": {
+ },"303": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Native plan string",
"content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-string",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-string"
- },"303": {
+ },"304": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Native plan with stats",
"content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-with-stats",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#native-plan-with-stats"
- },"304": {
+ },"305": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Gluten Implicits",
"content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-implicits",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/#gluten-implicits"
- },"305": {
+ },"306": {
"doc": "Getting Start with Velox Backend(v1.2.1)",
"title": "Getting Start with Velox Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/velox-backend/",
"relUrl": "/archives/v1.2.1/docs/getting-started/velox-backend/"
- },"306": {
+ },"307": {
"doc": "Using ABFS with Gluten(v1.2.1)",
"title": "Working with ABFS",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/abfs/#working-with-abfs",
"relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#working-with-abfs"
- },"307": {
+ },"308": {
"doc": "Using ABFS with Gluten(v1.2.1)",
"title": "Configuring ABFS Access Token",
"content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
"url": "/archives/v1.2.1/docs/getting-started/abfs/#configuring-abfs-access-token",
"relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#configuring-abfs-access-token"
- },"308": {
+ },"309": {
"doc": "Using ABFS with Gluten(v1.2.1)",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/abfs/#local-caching-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/abfs/#local-caching-support"
- },"309": {
+ },"310": {
"doc": "Using ABFS with Gluten(v1.2.1)",
"title": "Using ABFS with Gluten(v1.2.1)",
"content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
"url": "/archives/v1.2.1/docs/getting-started/abfs/",
"relUrl": "/archives/v1.2.1/docs/getting-started/abfs/"
- },"310": {
+ },"311": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Working with GCS",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#working-with-gcs",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#working-with-gcs"
- },"311": {
+ },"312": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Installing the gcloud CLI",
"content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#installing-the-gcloud-cli",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#installing-the-gcloud-cli"
- },"312": {
+ },"313": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS using a user account",
"content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-user-account",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-user-account"
- },"313": {
+ },"314": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS using a credential file",
"content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-using-a-credential-file"
- },"314": {
+ },"315": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS endpoints",
"content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-endpoints",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-endpoints"
- },"315": {
+ },"316": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS max retry count",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-count",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-count"
- },"316": {
+ },"317": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Configuring GCS max retry time",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retring (10s, 1m, etc). ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-time",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/#configuring-gcs-max-retry-time"
- },"317": {
+ },"318": {
"doc": "Using GCS with Gluten(v1.2.1)",
"title": "Using GCS with Gluten(v1.2.1)",
"content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.2.1/docs/getting-started/gcs/",
"relUrl": "/archives/v1.2.1/docs/getting-started/gcs/"
- },"318": {
+ },"319": {
"doc": "Velox Local Caching(v1.2.1)",
"title": "Velox Local Caching(v1.2.1)",
"content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
"url": "/archives/v1.2.1/docs/getting-started/localcache/",
"relUrl": "/archives/v1.2.1/docs/getting-started/localcache/"
- },"319": {
+ },"320": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Working with S3",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#working-with-s3",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#working-with-s3"
- },"320": {
+ },"321": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Configuring S3 endpoint",
"content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
"url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-endpoint",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-endpoint"
- },"321": {
+ },"322": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Configuring S3 instance credentials",
"content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-instance-credentials",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-instance-credentials"
- },"322": {
+ },"323": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Configuring S3 IAM roles",
"content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-iam-roles",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#configuring-s3-iam-roles"
- },"323": {
+ },"324": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Other authentatication methods are not supported yet",
"content": ". | AWS temporary credential | . ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#other-authentatication-methods-are-not-supported-yet"
- },"324": {
+ },"325": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Log granularity of AWS C++ SDK in velox",
"content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox"
- },"325": {
+ },"326": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/archives/v1.2.1/docs/getting-started/s3/#local-caching-support",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/#local-caching-support"
- },"326": {
+ },"327": {
"doc": "Using S3 with Gluten(v1.2.1)",
"title": "Using S3 with Gluten(v1.2.1)",
"content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.2.1/docs/getting-started/s3/",
"relUrl": "/archives/v1.2.1/docs/getting-started/s3/"
- },"327": {
+ },"328": {
"doc": "Build Parameters for Velox Backend(v1.2.1)",
"title": "Build Parameters",
"content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | OFF | . | build_proto [...]
"url": "/archives/v1.2.1/docs/getting-started/build-guide/#build-parameters",
"relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/#build-parameters"
- },"328": {
+ },"329": {
"doc": "Build Parameters for Velox Backend(v1.2.1)",
"title": "Gluten Jar for Deployment",
"content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . | 3.5.1 | 3.5 | 2.12 | . ",
"url": "/archives/v1.2.1/docs/getting-started/build-guide/#gluten-jar-for-deployment",
"relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/#gluten-jar-for-deployment"
- },"329": {
+ },"330": {
"doc": "Build Parameters for Velox Backend(v1.2.1)",
"title": "Build Parameters for Velox Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/build-guide/",
"relUrl": "/archives/v1.2.1/docs/getting-started/build-guide/"
- },"330": {
+ },"331": {
"doc": "Getting-Started(v1.2.1)",
"title": "Getting Started with Gluten for Apache Spark",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/#getting-started-with-gluten-for-apache-spark",
"relUrl": "/archives/v1.2.1/docs/getting-started/#getting-started-with-gluten-for-apache-spark"
- },"331": {
+ },"332": {
"doc": "Getting-Started(v1.2.1)",
"title": "Getting-Started(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/getting-started/",
"relUrl": "/archives/v1.2.1/docs/getting-started/"
- },"332": {
+ },"333": {
"doc": "Documentations(v1.2.1)",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.2.1/docs/#gluten-documents-by-version",
"relUrl": "/archives/v1.2.1/docs/#gluten-documents-by-version"
- },"333": {
+ },"334": {
"doc": "Documentations(v1.2.1)",
"title": "Documentations(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/",
"relUrl": "/archives/v1.2.1/docs/"
- },"334": {
+ },"335": {
"doc": "Velox Backend(v1.2.1)",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/#gluten-documents-by-version",
"relUrl": "/archives/v1.2.1/docs/velox-backend/#gluten-documents-by-version"
- },"335": {
+ },"336": {
"doc": "Velox Backend(v1.2.1)",
"title": "Velox Backend(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/"
- },"336": {
+ },"337": {
"doc": "Limitations(v1.2.1)",
"title": "Velox Backend Limitations",
"content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
"url": "/archives/v1.2.1/docs/velox-backend/limitations/#velox-backend-limitations",
"relUrl": "/archives/v1.2.1/docs/velox-backend/limitations/#velox-backend-limitations"
- },"337": {
+ },"338": {
"doc": "Limitations(v1.2.1)",
"title": "Limitations(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/limitations/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/limitations/"
- },"338": {
+ },"339": {
"doc": "Supported Operators & Functions(v1.2.1)",
"title": "The Operators and Functions Support Progress",
"content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. The total number of fun [...]
"url": "/archives/v1.2.1/docs/velox-backend/support/#the-operators-and-functions-support-progress",
"relUrl": "/archives/v1.2.1/docs/velox-backend/support/#the-operators-and-functions-support-progress"
- },"339": {
+ },"340": {
"doc": "Supported Operators & Functions(v1.2.1)",
"title": "Supported Operators & Functions(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/support/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/support/"
- },"340": {
+ },"341": {
"doc": "Troubleshooting(v1.2.1)",
"title": "Velox Backend Troubleshooting",
"content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
"url": "/archives/v1.2.1/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting",
"relUrl": "/archives/v1.2.1/docs/velox-backend/troubleshooting/#velox-backend-troubleshooting"
- },"341": {
+ },"342": {
"doc": "Troubleshooting(v1.2.1)",
"title": "Troubleshooting(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/troubleshooting/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/troubleshooting/"
- },"342": {
+ },"343": {
"doc": "Velox UDF(v1.2.1)",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"343": {
+ },"344": {
"doc": "Velox UDF(v1.2.1)",
"title": "Introduction",
"content": "Velox backend supports User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#introduction",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#introduction"
- },"344": {
+ },"345": {
"doc": "Velox UDF(v1.2.1)",
"title": "Create and Build UDF/UDAF library",
"content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
"url": "/archives/v1.2.1/docs/velox-backend/udf/#create-and-build-udfudaf-library",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#create-and-build-udfudaf-library"
- },"345": {
+ },"346": {
"doc": "Velox UDF(v1.2.1)",
"title": "Using UDF/UDAF in Gluten",
"content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the library paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
"url": "/archives/v1.2.1/docs/velox-backend/udf/#using-udfudaf-in-gluten",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#using-udfudaf-in-gluten"
- },"346": {
+ },"347": {
"doc": "Velox UDF(v1.2.1)",
"title": "Try the example",
"content": "We provided Velox UDF examples in file MyUDF.cc and UDAF examples in file MyUDAF.cc. You need to build the gluten project with --build_example=ON to get the example libraries./dev/buildbundle-veloxbe.sh --build_examples=ON . Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ . Start spark-shell or spark-sql with below configuration . # Use the `--files` option to upload a library and configure its relative path --files /path/to/glute [...]
"url": "/archives/v1.2.1/docs/velox-backend/udf/#try-the-example",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#try-the-example"
- },"347": {
+ },"348": {
"doc": "Velox UDF(v1.2.1)",
"title": "Configurations",
"content": "| Parameters | Description | . | spark.gluten.sql.columnar.backend.velox.udfLibraryPaths | Path to the udf/udaf libraries. | . | spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths | Path to the udf/udaf libraries on driver node. Only applicable on yarn-client mode. | . | spark.gluten.sql.columnar.backend.velox.udfAllowTypeConversion | Whether to inject possible cast to convert mismatched data types from input to one registered signatures. | . ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#configurations",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#configurations"
- },"348": {
+ },"349": {
"doc": "Velox UDF(v1.2.1)",
"title": "Pandas UDFs (a.k.a. Vectorized UDFs)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs"
- },"349": {
+ },"350": {
"doc": "Velox UDF(v1.2.1)",
"title": "Introduction",
"content": "Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer doc. ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/#introduction-1",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#introduction-1"
- },"350": {
+ },"351": {
"doc": "Velox UDF(v1.2.1)",
"title": "Using Pandas UDFs in Gluten with Velox Backend",
"content": "Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: . pip3 install pyspark==$SPARK_VERSION cython pip3 install pandas pyarrow . Gluten provides a config to control enable ColumnarArrowEvalPython or not, with true as defalt. spark.gluten.sql.columnar.arrowUdf . Then take following PySpark code for example: . from pyspark.sql.functions import pandas_udf, PandasUDFType import pyspark.sql.functions as [...]
"url": "/archives/v1.2.1/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend"
- },"351": {
+ },"352": {
"doc": "Velox UDF(v1.2.1)",
"title": "Velox UDF(v1.2.1)",
"content": " ",
"url": "/archives/v1.2.1/docs/velox-backend/udf/",
"relUrl": "/archives/v1.2.1/docs/velox-backend/udf/"
- },"352": {
+ },"353": {
"doc": "v1.2.1",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.2.1/#gluten-documents-by-version",
"relUrl": "/archives/v1.2.1/#gluten-documents-by-version"
- },"353": {
+ },"354": {
"doc": "v1.2.1",
"title": "v1.2.1",
"content": " ",
"url": "/archives/v1.2.1/",
"relUrl": "/archives/v1.2.1/"
- },"354": {
+ },"355": {
"doc": "Configuration",
"title": "Spark Configurations for Gluten Plugin",
"content": "There are many configurations could impact the Gluten Plugin performance and can be fine-tuned in Spark. You can add these configurations into spark-defaults.conf to enable or disable the setting. ",
"url": "/archives/v1.3.0/configuration#spark-configurations-for-gluten-plugin",
"relUrl": "/archives/v1.3.0/configuration#spark-configurations-for-gluten-plugin"
- },"355": {
+ },"356": {
"doc": "Configuration",
"title": "Spark parameters",
"content": "| Parameters | Description | Recommend Setting | . | spark.driver.extraClassPath | To add Gluten Plugin jar file in Spark Driver | /path/to/jar_file | . | spark.executor.extraClassPath | To add Gluten Plugin jar file in Spark Executor | /path/to/jar_file | . | spark.executor.memory | To set up how much memory to be used for Spark Executor. | | . | spark.memory.offHeap.size | To set up how much memory to be used for Java OffHeap. Please notice Gluten Plugin will leverage [...]
"url": "/archives/v1.3.0/configuration#spark-parameters",
"relUrl": "/archives/v1.3.0/configuration#spark-parameters"
- },"356": {
+ },"357": {
"doc": "Configuration",
"title": "Velox Parameters",
"content": "The following configurations are related to Velox settings. | Parameters | Description | Recommend Setting | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | . | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The m [...]
"url": "/archives/v1.3.0/configuration#velox-parameters",
"relUrl": "/archives/v1.3.0/configuration#velox-parameters"
- },"357": {
+ },"358": {
"doc": "Configuration",
"title": "Configuration",
"content": " ",
"url": "/archives/v1.3.0/configuration",
"relUrl": "/archives/v1.3.0/configuration"
- },"358": {
+ },"359": {
"doc": "CPP Code Style",
"title": "Gluten CPP Core Guidelines",
"content": "This is a set of CPP core guidelines for Gluten. The aim is to make the codebase simpler, more efficient, more maintainable by promoting consistency and according to best practices. ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#gluten-cpp-core-guidelines",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#gluten-cpp-core-guidelines"
- },"359": {
+ },"360": {
"doc": "CPP Code Style",
"title": "Philosophy",
"content": "Philosophical rules are generally not measurable. However, they are valuable. For Gluten CPP coding, there are a few Philosophical rules as the following. | Write in ISO Standard C++. | Standard API first, the CPP programming APIs are priority to system calls. | Write code consistently. It’s good for understanding and maintaining. | Keep simple, making code clear and easy to read. | Optimize code for reader, not for writer. Thus, more time will be spent reading code than [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#philosophy",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#philosophy"
- },"360": {
+ },"361": {
"doc": "CPP Code Style",
"title": "Code Formatting",
"content": "Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc). . | Always ensure your code is compatible with clang-format-12 for Velox backend. | dev/formatcppcode.sh is provided for formatting Velox CPP code. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#code-formatting",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#code-formatting"
- },"361": {
+ },"362": {
"doc": "CPP Code Style",
"title": "Naming Conventions",
"content": ". | Use PascalCase for types (class, struct, enum, type alias, type template parameter) and file name. | Use camelCase for function, member and local variable, and non-type template parameter. | Use camelCase_ for private and protected member variable. | Use snake_case for namespace name and build target. | Use UPPER_SNAKE_CASE for macro. | Use kPascalCase for static constant and enumerator. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#naming-conventions",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#naming-conventions"
- },"362": {
+ },"363": {
"doc": "CPP Code Style",
"title": "Designs",
"content": ". | No over design. | No negation of negation, isValid is better than isNotInvalid. | Avoid corner case, and common case first. | Express ideas directly, don’t let me think. | Make a check for the arguments in the interface between modules, and don’t make a check in the inner implementation, use assert in the private implementation instead of too much safe check. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#designs",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#designs"
- },"363": {
+ },"364": {
"doc": "CPP Code Style",
"title": "Source File & Header File",
"content": ". | All header files must have a single-inclusion guard using #pragma once | Always use .h as header file suffix, not .hpp. | Always use .cc as source file suffix, neither .cpp nor .cxx. | One file should contain one main class, and the file name should be consistent with the main class name. | Obvious exception: files used for defining various misc functions. | . | If a header file has a corresponding source file, they should have the same file name with different suffix [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#source-file--header-file",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#source-file--header-file"
- },"364": {
+ },"365": {
"doc": "CPP Code Style",
"title": "Class",
"content": ". | Base class name doesn’t end with Base, use Backend instead of BackendBase. | Ensure one class does one thing, and follows the single responsibility principle. | No big class, No huge class, No too much interfaces. | Distinguish interface from implementation, make implementations private. | When designing a class hierarchy, distinguish between interface inheritance and implementation inheritance. | Ensure that public inheritance represent the relation of is-a. | Ensure [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#class",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#class"
- },"365": {
+ },"366": {
"doc": "CPP Code Style",
"title": "Function",
"content": ". | Make functions short and simple. | Calling a meaningful function is more readable than writing too many statements in place, but the performance-sensitive code path is an exception. | Give the function a good name, how to check whether the function name is good or not. | When you read it loudly, you feel smooth. | The information can be represented by arguments should not be encoded into the function name. such as. use get(size_t index) instead of getByIndex. | . | A [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#function",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#function"
- },"366": {
+ },"367": {
"doc": "CPP Code Style",
"title": "Variable",
"content": ". | Make variable names simple and meaningful. | Don’t group all your variables at the top of the scope, it’s an outdated habit. | Declare variables as close to the usage point as possible. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#variable",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#variable"
- },"367": {
+ },"368": {
"doc": "CPP Code Style",
"title": "Constant",
"content": ". | Prefer const variables to using preprocessor (#define) to define constant values. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#constant",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#constant"
- },"368": {
+ },"369": {
"doc": "CPP Code Style",
"title": "Macro",
"content": ". | Macros downgrade readability, break mind, and affect debug. | Macros have side effects. | Use macros cautiously and carefully. | Consider using const variables or inline functions to replace macros. | Consider defining macros with the wrap of do {...} while (0) | Avoid using 3rd party library macros directly. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#macro",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#macro"
- },"369": {
+ },"370": {
"doc": "CPP Code Style",
"title": "Namespace",
"content": ". | Don’t using namespace xxx in header files. Instead, you can do this in source files. But it’s still not encouraged. | Place all Gluten CPP codes under namespace gluten because one level namespace is enough. No nested namespace. Nested namespaces bring mess. | The anonymous namespace is recommended for defining file level classes, functions and variables. It’s used to place file scoped static functions and variables. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#namespace",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#namespace"
- },"370": {
+ },"371": {
"doc": "CPP Code Style",
"title": "Resource Management",
"content": ". | Use handles and RAII to manage resources automatically. | Immediately give the result of an explicit resource allocation to a manager object. | Prefer scoped objects and stack objects. | Use raw pointers to denote individual objects. | Use pointer + size_t to denote array objects if you don’t want to use containers. | A raw pointer (a T*) is non-owning. | A raw reference (a T&) is non-owning. | Understand the difference of unique_ptr, shared_ptr, weak_ptr. | uniqu [...]
"url": "/archives/v1.3.0/developers/cpp-code-style/#resource-management",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#resource-management"
- },"371": {
+ },"372": {
"doc": "CPP Code Style",
"title": "Exception",
"content": ". | The exception specifications are changing always. The difference between various CPP standards is big, so we should use exception cautiously in Gluten. | Prefer return code to throwing exceptions. | Prefer compile-time checking to run-time checking. | Encapsulate messy constructors, rather than spreading through the code. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#exception",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#exception"
- },"372": {
+ },"373": {
"doc": "CPP Code Style",
"title": "Code Comment",
"content": ". | Add necessary comments. The comment is not the more the better, also not the less the better. | Good comment makes obscure code easily understood. It’s unnecessary to add comments for quite obvious code. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#code-comment",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#code-comment"
- },"373": {
+ },"374": {
"doc": "CPP Code Style",
"title": "References",
"content": ". | CppCoreGuidelines | Velox CODING_STYLE | Thanks Gluten developers for their wise suggestions and helps. | . ",
"url": "/archives/v1.3.0/developers/cpp-code-style/#references",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/#references"
- },"374": {
+ },"375": {
"doc": "CPP Code Style",
"title": "CPP Code Style",
"content": " ",
"url": "/archives/v1.3.0/developers/cpp-code-style/",
"relUrl": "/archives/v1.3.0/developers/cpp-code-style/"
- },"375": {
+ },"376": {
"doc": "How To Use Gluten",
"title": "How to understand the key work of Gluten?",
"content": "The Gluten worked as the role of bridge, it’s a middle layer between the Spark and the native execution library. The Gluten is responsibility for validating whether the operators of the Spark plan can be executed by the native engine or not. If yes, the Gluten transforms Spark plan to Substrait plan, and then send the Substrait plan to the native engine. The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. | All C++ codes are placed under the dir [...]
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-understand-the-key-work-of-gluten",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-understand-the-key-work-of-gluten"
- },"376": {
+ },"377": {
"doc": "How To Use Gluten",
"title": "How to debug in Gluten?",
"content": " ",
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-debug-in-gluten",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-debug-in-gluten"
- },"377": {
+ },"378": {
"doc": "How To Use Gluten",
"title": "1 How to debug C++",
"content": "If you don’t concern about the Scala/Java codes and just want to debug the C++ codes executed in native engine, you may debug the C++ via benchmarks with GDB. To debug C++, you have to generate the example files, the example files consist of: . | A file contained Substrait plan in JSON format | One or more input data files in Parquet format | . You can generate the example files by the following steps: . | build Velox and Gluten CPP gluten_home/dev/builddeps-veloxbe.sh -- [...]
"url": "/archives/v1.3.0/developers/how-to-use/#1-how-to-debug-c",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#1-how-to-debug-c"
- },"378": {
+ },"379": {
"doc": "How To Use Gluten",
"title": "2 How to debug plan validation process",
"content": "Gluten will validate generated plan before execute it, and validation usually happens in native side, so we provide a utility to help debug validation process in native side. | Run query with conf spark.gluten.sql.debug=true, and you will find generated plan be printed in stderr with json format, save it as plan.json for example. | Compile cpp part with --build_benchmarks=ON, then check plan_validator_util executable file in gluten_home/cpp/build/velox/benchmarks/. | Run [...]
"url": "/archives/v1.3.0/developers/how-to-use/#2-how-to-debug-plan-validation-process",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#2-how-to-debug-plan-validation-process"
- },"379": {
+ },"380": {
"doc": "How To Use Gluten",
"title": "3 How to debug Java/Scala",
"content": "wait to add . ",
"url": "/archives/v1.3.0/developers/how-to-use/#3-how-to-debug-javascala",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#3-how-to-debug-javascala"
- },"380": {
+ },"381": {
"doc": "How To Use Gluten",
"title": "4 How to debug with core-dump",
"content": "wait to complete . cd the_directory_of_core_file_generated gdb gluten_home/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . | the core-Executor task l-2000883-1671542526 represents the core file name. | . ",
"url": "/archives/v1.3.0/developers/how-to-use/#4-how-to-debug-with-core-dump",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#4-how-to-debug-with-core-dump"
- },"381": {
+ },"382": {
"doc": "How To Use Gluten",
"title": "How to run TPC-H on Velox backend",
"content": "Now, both Parquet and DWRF format files are supported, related scripts and files are under the directory of gluten_home/backends-velox/workload/tpch. The file README.md under gluten_home/backends-velox/workload/tpch offers some useful help but it’s still not enough and exact. One way of run TPC-H test is to run velox with docker by workflow, you can refer to velox_docker.yml . Here will explain how to run TPC-H on Velox backend with the Parquet file format. | First step, [...]
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-h-on-velox-backend"
- },"382": {
+ },"383": {
"doc": "How To Use Gluten",
"title": "How to run TPC-DS",
"content": "wait to add . ",
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-ds",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-run-tpc-ds"
- },"383": {
+ },"384": {
"doc": "How To Use Gluten",
"title": "How to track the memory exhaust problem",
"content": "When your gluten spark jobs failed because of OOM, you can track the memory allocation’s call stack by configuring spark.gluten.backtrace.allocation = true. The above configuration will use BacktraceAllocationListener wrapping from SparkAllocationListener to create VeloxMemoryManager. BacktraceAllocationListener will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3…G, the call stack of memory all [...]
"url": "/archives/v1.3.0/developers/how-to-use/#how-to-track-the-memory-exhaust-problem",
"relUrl": "/archives/v1.3.0/developers/how-to-use/#how-to-track-the-memory-exhaust-problem"
- },"384": {
+ },"385": {
"doc": "How To Use Gluten",
"title": "How To Use Gluten",
"content": "There are some common questions about developing, debugging and testing been asked again and again. In order to help the developers to contribute to Gluten as soon as possible, we collected these frequently asked questions, and organized them in the form of Q&A. It’s convenient for the developers to check and learn. when you encountered a new problem and then resolved it, please add a new item to this document if you think it may be helpful to the other developers. We [...]
"url": "/archives/v1.3.0/developers/how-to-use/",
"relUrl": "/archives/v1.3.0/developers/how-to-use/"
- },"385": {
+ },"386": {
"doc": "How To Release",
"title": "How to Release",
"content": "This section outlines the steps for releasing Apache Gluten (incubating) according to the Apache release guidelines. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/docs/developers/how-to-release/#how-to-release",
"relUrl": "/docs/developers/how-to-release/#how-to-release"
- },"386": {
+ },"387": {
"doc": "How To Release",
"title": "Release Process",
"content": ". | Prepare the release artifacts. | Upload the release artifacts to the SVN repository. | Verify the release artifacts. | Initiate a release vote. | Announce the results and the release. | . Prepare the release artifacts. | Create a branch from the target git repository. | Tag a RC and draft the release notes. | Build and Sign the release artifacts (including source archives, binaries, …etc). | Generate checksums for the artifacts. | . How to Sign the release artifacts. [...]
"url": "/docs/developers/how-to-release/#release-process",
"relUrl": "/docs/developers/how-to-release/#release-process"
- },"387": {
+ },"388": {
"doc": "How To Release",
"title": "How To Release",
"content": " ",
"url": "/docs/developers/how-to-release/",
"relUrl": "/docs/developers/how-to-release/"
- },"388": {
+ },"389": {
"doc": "How To Scan Security issues",
"title": "How to scan the security issues",
"content": "This section outlines the steps to use tools to scan Apache Gluten (incubating) source code and make sure no vulnerability issues in the code. All projects under the Apache umbrella must adhere to the Apache Release Policy. This guide is designed to assist you in comprehending the policy and navigating the process of releasing projects at Apache. ",
"url": "/archives/v1.3.0/developers/how-to-security-scan/#how-to-scan-the-security-issues",
"relUrl": "/archives/v1.3.0/developers/how-to-security-scan/#how-to-scan-the-security-issues"
- },"389": {
+ },"390": {
"doc": "How To Scan Security issues",
"title": "Scan Security Process",
"content": "Before every Apache Gluten (incubating) release, we need to ensure there is no vulnerability issue in the source code. We use Trivy as the tool to scan all the security issues. | Install Trivy, please follow the steps to install Trivy: Trivy Installation . | Configuring Trivy, please follow the guide to configure Trivy for specific operation: Trivy Configuration . | Run Trivy File System Scan with the source code. Below is an example about how we run Trivy scan with Apach [...]
"url": "/archives/v1.3.0/developers/how-to-security-scan/#scan-security-process",
"relUrl": "/archives/v1.3.0/developers/how-to-security-scan/#scan-security-process"
- },"390": {
+ },"391": {
"doc": "How To Scan Security issues",
"title": "How To Scan Security issues",
"content": " ",
"url": "/archives/v1.3.0/developers/how-to-security-scan/",
"relUrl": "/archives/v1.3.0/developers/how-to-security-scan/"
- },"391": {
+ },"392": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Generate Micro Benchmarks for Velox Backend",
"content": "This document explains how to use the existing micro benchmark template in Gluten Cpp. A micro benchmark for Velox backend is provided in Gluten Cpp to simulate the execution of a first or middle stage in Spark. It serves as a more convenient alternative to debug in Gluten Cpp comparing with directly debugging in a Spark job. Developers can use it to create their own workloads, debug in native process, profile the hotspot and do optimizations. To simulate a first stage, y [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#generate-micro-benchmarks-for-velox-backend"
- },"392": {
+ },"393": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Try the example",
"content": "To run a micro benchmark, user should provide one file that contains the Substrait plan in JSON format, and optional one or more input data files in parquet format. The commands below help to generate example input files: . cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON # Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.2 with -Pspark-3.3 mvn test -Pspark-3.2 -Pbackends-velox -Prss -pl backends-v [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#try-the-example",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#try-the-example"
- },"393": {
+ },"394": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Generate Substrait plan and input for any query",
"content": "First, build Gluten with --build_benchmarks=ON. cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_benchmarks=ON # For debugging purpose, rebuild Gluten with build type `Debug`./dev/buildbundle-veloxbe.sh --build_benchmarks=ON --build_type=Debug . First, get the Stage Id from spark UI for the stage you want to simulate. And then re-run the query with below configurations to dump the inputs to micro benchmark. | Parameters | Description | Recommend Setting | . | spar [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#generate-substrait-plan-and-input-for-any-query"
- },"394": {
+ },"395": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Save ouput to parquet to analyze",
"content": "You can save the output to a parquet file to analyze. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --data /absolute_path/to/data.parquet --threads 1 --noprint-result --write-file=/absolute_path/to/result.parquet . ",
"url": "/archives/v1.3.0/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#save-ouput-to-parquet-to-analyze"
- },"395": {
+ },"396": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Add shuffle write process",
"content": "You can add the shuffle write process at the end of this stage. Note that this will ignore the --write-file option. cd /path/to/gluten/cpp/build/velox/benchmarks ./generic_benchmark \\ --plan /absolute_path/to/plan.json \\ --split /absolute_path/to/split.json \\ --threads 1 --noprint-result --with-shuffle . By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by adding one of the following argument flags to the command: . | –zstd: Z [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#add-shuffle-write-process",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#add-shuffle-write-process"
- },"396": {
+ },"397": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Simulate Spark with multiple processes and threads",
"content": "You can use below command to launch several processes and threads to simulate parallel execution on Spark. Each thread in the same process will be pinned to the core number starting from --cpu. Suppose running on a baremetal machine with 48C, 2-socket, HT-on, launching below command will utilize all vcores. processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do ./generic_benchmark --plan [...]
"url": "/archives/v1.3.0/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/#simulate-spark-with-multiple-processes-and-threads"
- },"397": {
+ },"398": {
"doc": "Micro Benchmarks for Velox Backend",
"title": "Micro Benchmarks for Velox Backend",
"content": " ",
"url": "/archives/v1.3.0/developers/microbenchmarks/",
"relUrl": "/archives/v1.3.0/developers/microbenchmarks/"
- },"398": {
+ },"399": {
"doc": "New To Gluten",
"title": "Environment",
"content": "Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. ",
"url": "/archives/v1.3.0/developers/new-to/#environment",
"relUrl": "/archives/v1.3.0/developers/new-to/#environment"
- },"399": {
+ },"400": {
"doc": "New To Gluten",
"title": "Openjdk8",
"content": "Environment setting . For root user, the environment variables file is /etc/profile, it will make effect for all the users. For other user, you can set in ~/.bashrc. Guide for ubuntu . The default JDK version in ubuntu is java11, we need to set to java8. apt install openjdk-8-jdk update-alternatives --config java java -version . --config java to config java executable path, javac and other commands can also use this command to config. For some other uses, we suggest to se [...]
"url": "/archives/v1.3.0/developers/new-to/#openjdk8",
"relUrl": "/archives/v1.3.0/developers/new-to/#openjdk8"
- },"400": {
+ },"401": {
"doc": "New To Gluten",
"title": "Maven 3.6.3 or above",
"content": "Maven Dowload Page And then set the environment setting. ",
"url": "/archives/v1.3.0/developers/new-to/#maven-363-or-above",
"relUrl": "/archives/v1.3.0/developers/new-to/#maven-363-or-above"
- },"401": {
+ },"402": {
"doc": "New To Gluten",
"title": "GCC 9.4 or above",
"content": " ",
"url": "/archives/v1.3.0/developers/new-to/#gcc-94-or-above",
"relUrl": "/archives/v1.3.0/developers/new-to/#gcc-94-or-above"
- },"402": {
+ },"403": {
"doc": "New To Gluten",
"title": "Compile gluten using debug mode",
"content": "If you want to just debug java/scala code, there is no need to compile cpp code with debug mode. You can just refer to build-gluten-with-velox-backend. If you need to debug cpp code, please compile the backend code and gluten cpp code with debug mode. ## compile velox backend with benchmark and tests to debug gluten_home/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --build_type=Debug . If you need to debug the tests in /gluten-ut, You need to compile ja [...]
"url": "/archives/v1.3.0/developers/new-to/#compile-gluten-using-debug-mode",
"relUrl": "/archives/v1.3.0/developers/new-to/#compile-gluten-using-debug-mode"
- },"403": {
+ },"404": {
"doc": "New To Gluten",
"title": "Java/scala code development with Intellij",
"content": " ",
"url": "/archives/v1.3.0/developers/new-to/#javascala-code-development-with-intellij",
"relUrl": "/archives/v1.3.0/developers/new-to/#javascala-code-development-with-intellij"
- },"404": {
+ },"405": {
"doc": "New To Gluten",
"title": "Linux intellij local debug",
"content": "Install the linux intellij version, and debug code locally. | Ask your linux maintainer to install the desktop, and then restart the server. | If you use Moba-XTerm to connect linux server, you don’t need to install x11 server, If not (e.g. putty), please follow this guide: X11 Forwarding: Setup Instructions for Linux and Mac . | Download intellij linux community version to linux server | Start Idea, bash <idea_dir>/idea.sh | . Notes: Sometimes, your desktop may sto [...]
"url": "/archives/v1.3.0/developers/new-to/#linux-intellij-local-debug",
"relUrl": "/archives/v1.3.0/developers/new-to/#linux-intellij-local-debug"
- },"405": {
+ },"406": {
"doc": "New To Gluten",
"title": "Windows/Mac intellij remote debug",
"content": "If you have Ultimate intellij, you can try to debug remotely. ",
"url": "/archives/v1.3.0/developers/new-to/#windowsmac-intellij-remote-debug",
"relUrl": "/archives/v1.3.0/developers/new-to/#windowsmac-intellij-remote-debug"
- },"406": {
+ },"407": {
"doc": "New To Gluten",
"title": "Set up gluten project",
"content": ". | Make sure you have compiled gluten. | Load the gluten by File->Open, select <gluten_home/pom.xml>. | Activate your profiles such as , and Reload Maven Project, you will find all your need modules have been activated. | Create breakpoint and debug as you wish, maybe you can try CTRL+N to find TestOperator to start your test. | . ",
"url": "/archives/v1.3.0/developers/new-to/#set-up-gluten-project",
"relUrl": "/archives/v1.3.0/developers/new-to/#set-up-gluten-project"
- },"407": {
+ },"408": {
"doc": "New To Gluten",
"title": "Java/Scala code style",
"content": "Intellij IDE supports importing settings for Java/Scala code style. You can import intellij-codestyle.xml to your IDE. See Intellij guide. To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. For Velox backend: . mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.2 -Pspark-ut -DskipTests mvn spotless:apply -Pbackends-velox -Prss -Pspark-3.3 -Pspark-ut -DskipTests . For Clickhouse bac [...]
"url": "/archives/v1.3.0/developers/new-to/#javascala-code-style",
"relUrl": "/archives/v1.3.0/developers/new-to/#javascala-code-style"
- },"408": {
+ },"409": {
"doc": "New To Gluten",
"title": "CPP code development with Visual Studio Code",
"content": "This guide is for remote debug. We will connect the remote linux server by SSH. Download the windows vscode software The important leftside bar is: . | Explorer (Project structure) | Search | Run and Debug | Extensions (Install C/C++ Extension Pack, Remote Development, GitLens at least, C++ Test Mate is also suggested) | Remote Explorer (Connect linux server by ssh command, click +, then input ssh user@10.1.7.003) | Manage (Settings) | . Input your password in the above p [...]
"url": "/archives/v1.3.0/developers/new-to/#cpp-code-development-with-visual-studio-code",
"relUrl": "/archives/v1.3.0/developers/new-to/#cpp-code-development-with-visual-studio-code"
- },"409": {
+ },"410": {
"doc": "New To Gluten",
"title": "Usage",
"content": "Set up project . File->Open Folder // select gluten folder Select cpp/CmakeList.txt as command prompt Select gcc version as command prompt . Settings . VSCode support 2 ways to set user setting. | Manage->Command Palette(Open settings.json, search by Preferences: Open Settings (JSON)) | Manage->Settings (Common setting) | . Build by vscode . VSCode will try to compile the debug version in /build. And we need to compile velox debug mode before, if you have compile [...]
"url": "/archives/v1.3.0/developers/new-to/#usage",
"relUrl": "/archives/v1.3.0/developers/new-to/#usage"
- },"410": {
+ },"411": {
"doc": "New To Gluten",
"title": "Usefule notes",
"content": "Upgrade vscode . No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off Search update in Manage->Settings to turn off update mode . Colour setting . \"workbench.colorTheme\": \"Quiet Light\", \"files.autoSave\": \"afterDelay\", \"workbench.colorCustomizations\": { \"editor.wordHighlightBackground\": \"#063ef7\", // \"editor.selectionBackground\": \"#d1d1c6\", // \"tab.activeBackground\": \"#b8b9988c\", \"editor.selec [...]
"url": "/archives/v1.3.0/developers/new-to/#usefule-notes",
"relUrl": "/archives/v1.3.0/developers/new-to/#usefule-notes"
- },"411": {
+ },"412": {
"doc": "New To Gluten",
"title": "Debug cpp code with coredump",
"content": "mkdir -p /mnt/DP_disk1/core sysctl -w kernel.core_pattern=/mnt/DP_disk1/core/core-%e-%p-%t cat /proc/sys/kernel/core_pattern # set the core file to unlimited size echo \"ulimit -c unlimited\" >> ~/.bashrc # then you will get the core file at `/mnt/DP_disk1/core` when the program crashes # gdb -c corefile # gdb <gluten_home>/cpp/build/releases/libgluten.so 'core-Executor task l-2000883-1671542526' . ‘core-Executor task l-2000883-1671542526’ is the generated cor [...]
"url": "/archives/v1.3.0/developers/new-to/#debug-cpp-code-with-coredump",
"relUrl": "/archives/v1.3.0/developers/new-to/#debug-cpp-code-with-coredump"
- },"412": {
+ },"413": {
"doc": "New To Gluten",
"title": "Debug cpp with gdb",
"content": "You can use gdb to debug tests and benchmarks. And also you can debug jni call. Place the following code to your debug path. pid_t pid = getpid(); printf(\"----------------------------------pid: %lun\", pid); sleep(10); . You can also get the pid by java command or grep java program when executing unit test. jps 1375551 ScalaTestRunner ps ux | grep TestOperator . Execute gdb command to debug: . gdb attach <pid> . gdb attach 1375551 wait to attach.... (gdb) b <vel [...]
"url": "/archives/v1.3.0/developers/new-to/#debug-cpp-with-gdb",
"relUrl": "/archives/v1.3.0/developers/new-to/#debug-cpp-with-gdb"
- },"413": {
+ },"414": {
"doc": "New To Gluten",
"title": "Run TPC-H and TPC-DS",
"content": "We supply <gluten_home>/tools/gluten-it to execute these queries Refer to velox_docker.yml . ",
"url": "/archives/v1.3.0/developers/new-to/#run-tpc-h-and-tpc-ds",
"relUrl": "/archives/v1.3.0/developers/new-to/#run-tpc-h-and-tpc-ds"
- },"414": {
+ },"415": {
"doc": "New To Gluten",
"title": "Run gluten+velox on clean machine",
"content": "We can run gluten + velox on clean machine by one command (supported OS: Ubuntu20.04/22.04, Centos 7/8, etc.). spark-shell --name run_gluten \\ --master yarn --deploy-mode client \\ --conf spark.plugins=io.glutenproject.GlutenPlugin \\ --conf spark.memory.offHeap.enabled=true \\ --conf spark.memory.offHeap.size=20g \\ --jars https://github.com/oap-project/gluten/releases/download/v1.0.0/gluten-velox-bundle-spark3.2_2.12-ubuntu_20.04_x86_64-1.0.0.jar \\ --conf spark.shuffl [...]
"url": "/archives/v1.3.0/developers/new-to/#run-glutenvelox-on-clean-machine",
"relUrl": "/archives/v1.3.0/developers/new-to/#run-glutenvelox-on-clean-machine"
- },"415": {
+ },"416": {
"doc": "New To Gluten",
"title": "New To Gluten",
"content": "Help users to debug and test with gluten. For intel internal developer, you could refer to internal wiki New Employee Guide to get more information such as proxy settings, Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. ",
"url": "/archives/v1.3.0/developers/new-to/",
"relUrl": "/archives/v1.3.0/developers/new-to/"
- },"416": {
+ },"417": {
"doc": "Substrait Modifications",
"title": "Substrait Modifications in Gluten",
"content": "Substrait is a project aiming to create a well-defined, cross-language specification for data compute operations. Since it is still under active development, there are some lacking representations for Gluten needed computing operations. At the same time, some existing representations need to be modified a bit to satisfy the needs of computing. In Gluten, the base version of Substrait is v0.23.0. This page records all the Gluten changes to Substrait proto files for referen [...]
"url": "/archives/v1.3.0/developers/substrait/#substrait-modifications-in-gluten",
"relUrl": "/archives/v1.3.0/developers/substrait/#substrait-modifications-in-gluten"
- },"417": {
+ },"418": {
"doc": "Substrait Modifications",
"title": "Modifications to algebra.proto",
"content": ". | Added JsonReadOptions and TextReadOptions in FileOrFiles(#1584). | Changed join type JOIN_TYPE_SEMI to JOIN_TYPE_LEFT_SEMI and JOIN_TYPE_RIGHT_SEMI(#408). | Added WindowRel, added column_name and window_type in WindowFunction, changed Unbounded in WindowFunction into Unbounded_Preceding and Unbounded_Following, and added WindowType(#485). | Added output_schema in RelRoot(#1901). | Added ExpandRel(#1361). | Added GenerateRel(#574). | Added PartitionColumn in LocalFiles [...]
"url": "/archives/v1.3.0/developers/substrait/#modifications-to-algebraproto",
"relUrl": "/archives/v1.3.0/developers/substrait/#modifications-to-algebraproto"
- },"418": {
+ },"419": {
"doc": "Substrait Modifications",
"title": "Modifications to type.proto",
"content": ". | Added Nothing in Type(#791). | Added names in Struct(#1878). | Added PartitionColumns in NamedStruct(#320). | Remove PartitionColumns and add column_types in NamedStruct(#2405). | . ",
"url": "/archives/v1.3.0/developers/substrait/#modifications-to-typeproto",
"relUrl": "/archives/v1.3.0/developers/substrait/#modifications-to-typeproto"
- },"419": {
+ },"420": {
"doc": "Substrait Modifications",
"title": "Substrait Modifications",
"content": " ",
"url": "/archives/v1.3.0/developers/substrait/",
"relUrl": "/archives/v1.3.0/developers/substrait/"
- },"420": {
+ },"421": {
"doc": "Docker script for CentOS 7",
"title": "Docker script for CentOS 7",
"content": "Here is a docker script we verified to build Gluten+Velox backend on CentOS 7: . Run on host as root user: . docker pull centos:7 docker run -itd --name gluten centos:7 /bin/bash docker attach gluten . Run in docker: . yum -y install epel-release centos-release-scl yum -y install \\ git236 \\ dnf \\ cmake3 \\ devtoolset-9 \\ java-1.8.0-openjdk \\ java-1.8.0-openjdk-devel \\ ninja-build \\ wget \\ autoconf \\ autoconf-archive \\ automake \\ perl-IPC-Cmd \\ libicu-devel \\ [...]
"url": "/archives/v1.3.0/developers/docker-centos7/",
"relUrl": "/archives/v1.3.0/developers/docker-centos7/"
- },"421": {
+ },"422": {
"doc": "Docker script for CentOS 8",
"title": "Docker script for CentOS 8",
"content": "Here is a docker script we verified to build Gluten+Velox backend on Centos8: . Run on host as root user: . docker pull centos:8 docker run -itd --name gluten centos:8 /bin/bash docker attach gluten . Run in docker: . #update mirror sed -i -e \"s|mirrorlist=|#mirrorlist=|g\" /etc/yum.repos.d/CentOS-* sed -i -e \"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g\" /etc/yum.repos.d/CentOS-* dnf install -y epel-release sudo yum install -y dnf-plugins-core [...]
"url": "/archives/v1.3.0/developers/docker-centos8/",
"relUrl": "/archives/v1.3.0/developers/docker-centos8/"
- },"422": {
+ },"423": {
"doc": "Docker script for Ubuntu 22.04/20.04",
"title": "Docker script for Ubuntu 22.04/20.04",
"content": "To the first build, it’s suggested to build Gluten in a clean docker image. Otherwise it’s easy to run into library version conflict issues. Here is a docker script we verified to build Gluten+Velox backend on Ubuntu22.04/20.04: . Run on host as root user: . docker pull ubuntu:22.04 docker run -itd --network host --name gluten ubuntu:22.04 /bin/bash docker attach gluten . Run in docker: . apt-get update #install gcc and libraries to build arrow apt install software-proper [...]
"url": "/archives/v1.3.0/developers/docker-ubuntu/",
"relUrl": "/archives/v1.3.0/developers/docker-ubuntu/"
- },"423": {
+ },"424": {
"doc": "Developers",
"title": "Gluten Developers",
"content": "This document provides a developer overview of the project and covers the following topics: . ",
"url": "/archives/v1.3.0/developers#gluten-developers",
"relUrl": "/archives/v1.3.0/developers#gluten-developers"
- },"424": {
+ },"425": {
"doc": "Developers",
"title": "Developers",
"content": " ",
"url": "/archives/v1.3.0/developers",
"relUrl": "/archives/v1.3.0/developers"
- },"425": {
+ },"426": {
"doc": "Velox Function Development",
"title": "Developer Guide for Implementing Spark Built-in SQL Functions in Velox",
"content": "In velox, two folders prestosql & sparksql are holding most sql functions, respective for presto and spark. Gluten will ask velox to firstly register prestosql functions, then sparksql functions. So if prestosql and sparksql share same signature for a function, the sparksql function will overwrite the corresponding prestosql function. If the required function is lacking in both folders (exceptions are some common functions defined outside, like cast), we need to imple [...]
"url": "/archives/v1.3.0/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox",
"relUrl": "/archives/v1.3.0/developers/velox-function-dev/#developer-guide-for-implementing-spark-built-in-sql-functions-in-velox"
- },"426": {
+ },"427": {
"doc": "Velox Function Development",
"title": "Velox Function Development",
"content": " ",
"url": "/archives/v1.3.0/developers/velox-function-dev/",
"relUrl": "/archives/v1.3.0/developers/velox-function-dev/"
- },"427": {
+ },"428": {
"doc": "Getting start with ClickHouse Backend",
"title": "ClickHouse Backend",
"content": "ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP), which supports best in the industry query performance, while significantly reducing storage requirements through its innovative use of columnar storage and compression. We port ClickHouse ( based on version 23.1 ) as a library, called ‘libch.so’, and Gluten loads this library through JNI as the native engine. In this way, we don’t need to deploy a standalo [...]
"url": "/archives/v1.3.0/getting-started/clickhouse-backend/#clickhouse-backend",
"relUrl": "/archives/v1.3.0/getting-started/clickhouse-backend/#clickhouse-backend"
- },"428": {
+ },"429": {
"doc": "Getting start with ClickHouse Backend",
"title": "Getting start with ClickHouse Backend",
"content": " ",
"url": "/archives/v1.3.0/getting-started/clickhouse-backend/",
"relUrl": "/archives/v1.3.0/getting-started/clickhouse-backend/"
- },"429": {
+ },"430": {
"doc": "Getting Start with Velox Backend",
"title": "Supported Version",
"content": "| Type | Version | . | Spark | 3.2.2, 3.3.1, 3.4.2, 3.5.1 | . | OS | Ubuntu20.04/22.04, Centos7/8 | . | jdk | openjdk8/jdk17 | . | scala | 2.12 | . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#supported-version",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#supported-version"
- },"430": {
+ },"431": {
"doc": "Getting Start with Velox Backend",
"title": "Prerequisite",
"content": "Currently, Gluten+Velox backend is only tested on Ubuntu20.04/Ubuntu22.04/Centos7/Centos8. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.2 and 3.5.1. We need to set up the JAVA_HOME env. Currently, Gluten supports java 8 and java 17. For x86_64 . ## make sure jdk8 is used export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#prerequisite",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#prerequisite"
- },"431": {
+ },"432": {
"doc": "Getting Start with Velox Backend",
"title": "Build Gluten with Velox Backend",
"content": "It’s recommended to use buildbundle-veloxbe.sh to build gluten in one script. Gluten build guide listed the parameters and their default value of build command for your reference. For x86_64 build . First time build for all supported spark versions./dev/buildbundle-veloxbe.sh . After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and setting up build dependencies./dev/buildbundle-veloxbe.sh --enable_ep [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-velox-backend",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-velox-backend"
- },"432": {
+ },"433": {
"doc": "Getting Start with Velox Backend",
"title": "Dependency library deployment",
"content": "With config enable_vcpkg=ON, the dependency libraries will be built and statically linked into libvelox.so and libgluten.so, which is packed into the gluten-jar. In this way, only the gluten-jar is needed to add to spark.<driver|executor>.extraClassPath and spark will deploy the jar to each worker node. It’s better to build the static version using a clean docker image without any extra libraries installed. On host with some libraries like jemalloc installed, the sc [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#dependency-library-deployment",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#dependency-library-deployment"
- },"433": {
+ },"434": {
"doc": "Getting Start with Velox Backend",
"title": "HDFS support",
"content": "Hadoop hdfs support is ready via the libhdfs3 library. The libhdfs3 provides native API for Hadoop I/O without the drawbacks of JNI. It also provides advanced authentication like Kerberos based. Please note this library has several dependencies which may require extra installations on Driver and Worker node. Build with HDFS support . To build Gluten with HDFS support, below command is suggested: . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_hdfs=ON . Configur [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#hdfs-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#hdfs-support"
- },"434": {
+ },"435": {
"doc": "Getting Start with Velox Backend",
"title": "Azure Blob File System (ABFS) support",
"content": "Velox supports ABFS with the open source Azure SDK for C++ and Gluten uses the Velox ABFS connector to connect with ABFS. The build option for ABFS (enable_abfs) must be set to enable this feature as listed below. cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_abfs=ON . Please refer Velox ABFS part for more detailed configurations. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#azure-blob-file-system-abfs-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#azure-blob-file-system-abfs-support"
- },"435": {
+ },"436": {
"doc": "Getting Start with Velox Backend",
"title": "AWS S3 support",
"content": "Velox supports S3 with the open source AWS C++ SDK and Gluten uses Velox S3 connector to connect with S3. A new build option for S3(enable_s3) is added. Below command is used to enable this feature . cd /path/to/gluten ./dev/buildbundle-veloxbe.sh --enable_s3=ON . Currently there are several ways to asscess S3 in Spark. Please refer Velox S3 part for more detailed configurations . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#aws-s3-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#aws-s3-support"
- },"436": {
+ },"437": {
"doc": "Getting Start with Velox Backend",
"title": "Celeborn support",
"content": "Gluten with velox backend supports Celeborn as remote shuffle service. Currently, the supported Celeborn versions are 0.3.x, 0.4.x and 0.5.0. Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. When compiling the Gluten Java module, it’s required to enable celeborn profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests . Then add the Gluten and Spa [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#celeborn-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#celeborn-support"
- },"437": {
+ },"438": {
"doc": "Getting Start with Velox Backend",
"title": "Uniffle support",
"content": "Uniffle with velox backend supports Uniffle as remote shuffle service. Currently, the supported Uniffle versions are 0.8.0. First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with uniffle. When compiling the Gluten Java module, it’s required to enable uniffle profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests . Then add the Uniffle and Spark Celeborn Client packages to your Spark application’s classpath(usu [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#uniffle-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#uniffle-support"
- },"438": {
+ },"439": {
"doc": "Getting Start with Velox Backend",
"title": "DeltaLake Support",
"content": "Gluten with velox backend supports DeltaLake table. How to use . First of all, compile gluten-delta module by a delta profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests . Then, put the additional gluten-delta-XX-SNAPSHOT.jar to the class path (usually it’s $SPARK_HOME/jars). The gluten-delta jar is in gluten-delta/target directory. After the two steps, you can query delta table by gluten/velox without scan’s fallback. Gluten with vel [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#deltalake-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#deltalake-support"
- },"439": {
+ },"440": {
"doc": "Getting Start with Velox Backend",
"title": "Iceberg Support",
"content": "Gluten with velox backend supports Iceberg table. Currently, only reading COW (Copy-On-Write) tables is supported. How to use . First of all, compile gluten-iceberg module by a iceberg profile, as follows: . mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests . Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan’s fallback. After the two steps, you can query [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#iceberg-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#iceberg-support"
- },"440": {
+ },"441": {
"doc": "Getting Start with Velox Backend",
"title": "Coverage",
"content": "Spark3.3 has 387 functions in total. ~240 are commonly used. To get the support status of all Spark built-in functions, please refer to Velox Backend’s Supported Operators & Functions. Velox doesn’t support ANSI mode), so as Gluten. Once ANSI mode is enabled in Spark config, Gluten will fallback to Vanilla Spark. To identify what can be offloaded in a query and detailed fallback reasons, user can follow below steps to retrieve corresponding logs. 1) Enable Gluten by p [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#coverage",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#coverage"
- },"441": {
+ },"442": {
"doc": "Getting Start with Velox Backend",
"title": "Spill (Experimental)",
"content": "Velox backend supports spilling-to-disk. Using the following configuration options to customize spilling: . | Name | Default Value | Description | . | spark.gluten.sql.columnar.backend.velox.spillStrategy | auto | none: Disable spill on Velox backend; auto: Let Spark memory manager manage Velox’s spilling | . | spark.gluten.sql.columnar.backend.velox.spillFileSystem | local | The filesystem used to store spill data. local: The local file system. heap-over-local: Write fil [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#spill-experimental",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#spill-experimental"
- },"442": {
+ },"443": {
"doc": "Getting Start with Velox Backend",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": "Please check the VeloxNativeUDF.md for more detailed usage and configurations. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"443": {
+ },"444": {
"doc": "Getting Start with Velox Backend",
"title": "High-Bandwidth Memory (HBM) support",
"content": "Gluten supports allocating memory on HBM. This feature is optional and is disabled by default. It is implemented on top of Memkind library. You can refer to memkind’s readme for more details. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#high-bandwidth-memory-hbm-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#high-bandwidth-memory-hbm-support"
- },"444": {
+ },"445": {
"doc": "Getting Start with Velox Backend",
"title": "Build Gluten with HBM",
"content": "Gluten will internally build and link to a specific version of Memkind library and hwloc. Other dependencies should be installed on Driver and Worker node first: . sudo apt install -y autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev . After the set-up, you can now build Gluten with HBM. Below command is used to enable this feature . cd /path/to/gluten ## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.2 and 3.5.1./dev/buildbundle-veloxbe.sh -- [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-hbm",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-hbm"
- },"445": {
+ },"446": {
"doc": "Getting Start with Velox Backend",
"title": "Configure and enable HBM in Spark Application",
"content": "At runtime, MEMKIND_HBW_NODES enviroment variable is detected for configuring HBM NUMA nodes. For the explaination to this variable, please refer to memkind’s manual page. This can be set for all executors through spark conf, e.g. --conf spark.executorEnv.MEMKIND_HBW_NODES=8-15. Note that memory allocation fallback is also supported and cannot be turned off. If HBM is unavailable or fills up, the allocator will use default(DDR) memory. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#configure-and-enable-hbm-in-spark-application"
- },"446": {
+ },"447": {
"doc": "Getting Start with Velox Backend",
"title": "Intel® QuickAssist Technology (QAT) support",
"content": "Gluten supports using Intel® QuickAssist Technology (QAT) for data compression during Spark Shuffle. It benefits from QAT Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on QAT driver library and QATzip library. Please manually download QAT driver for your system, and follow its README to build and install on all Driver [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#intel-quickassist-technology-qat-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#intel-quickassist-technology-qat-support"
- },"447": {
+ },"448": {
"doc": "Getting Start with Velox Backend",
"title": "Software Requirements",
"content": ". | Download QAT driver for your system, and follow its README to build and install on all Driver and Worker nodes: Intel® QuickAssist Technology Driver for Linux* – HW Version 2.0. | Below compression libraries need to be installed on all Driver and Worker nodes: . | Zlib* library of version 1.2.7 or higher | ZSTD* library of version 1.5.4 or higher | LZ4* library | . | . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#software-requirements",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#software-requirements"
- },"448": {
+ },"449": {
"doc": "Getting Start with Velox Backend",
"title": "Build Gluten with QAT",
"content": ". | Setup ICP_ROOT environment variable to the directory where QAT driver is extracted. This environment variable is required during building Gluten and running Spark applications. It’s recommended to put it in .bashrc on Driver and Worker nodes. | . echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc source ~/.bashrc # Also set for root if running as non-root user sudo su - echo \"export ICP_ROOT=/path/to/QAT_driver\" >> ~/.bashrc exit . | This step is [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-qat",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-qat"
- },"449": {
+ },"450": {
"doc": "Getting Start with Velox Backend",
"title": "Enable QAT with Gzip/Zstd for shuffle compression",
"content": ". | To offload shuffle compression into QAT, first make sure you have the right QAT configuration file at /etc/4xxx_devX.conf. We provide a example configuration file. This configuration sets up to 4 processes that can bind to 1 QAT, and each process can use up to 16 QAT DC instances. | . ## run as root ## Overwrite QAT configuration file. cd /etc for i in {0..7}; do echo \"4xxx_dev$i.conf\"; done | xargs -i cp -f /path/to/gluten/docs/qat/4x16.conf {} ## Restart QAT after [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#enable-qat-with-gzipzstd-for-shuffle-compression"
- },"450": {
+ },"451": {
"doc": "Getting Start with Velox Backend",
"title": "QAT driver references",
"content": "Documentation . README Text Files (README_QAT20.L.1.0.0-00021.txt) . Release Notes . Check out the Intel® QuickAssist Technology Software for Linux* - Release Notes for the latest changes in this release. Getting Started Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Getting Started Guide for detailed installation instructions. Programmer’s Guide . Check out the Intel® QuickAssist Technology Software for Linux* - Programmer’s Guide for software [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#qat-driver-references",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#qat-driver-references"
- },"451": {
+ },"452": {
"doc": "Getting Start with Velox Backend",
"title": "Intel® In-memory Analytics Accelerator (IAA/IAX) support",
"content": "Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics Accelerator (IAA, also called IAX) for data compression during Spark Shuffle. It benefits from IAA Hardware-based acceleration on compression/decompression, and uses Gzip as compression format for higher compression ratio to reduce the pressure on disks and network transmission. This feature is based on Intel® QPL. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#intel-in-memory-analytics-accelerator-iaaiax-support"
- },"452": {
+ },"453": {
"doc": "Getting Start with Velox Backend",
"title": "Build Gluten with IAA",
"content": "Gluten will internally build and link to a specific version of QPL library, but extra environment setup is still required. Please refer to QPL Installation Guide to install dependencies and configure accelerators. This step is required if your application is running as Non-root user. Create a group for the users who have privilege to use IAA, and grant group iaa read/write access to the IAA Work-Queues. sudo groupadd iaa sudo usermod -aG iaa username # need to relogin sud [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-iaa",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#build-gluten-with-iaa"
- },"453": {
+ },"454": {
"doc": "Getting Start with Velox Backend",
"title": "Enable IAA with Gzip Compression for shuffle compression",
"content": ". | To enable QAT at run-time, first make sure you have configured the IAA Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct. | . sudo ls -l /dev/iax . The output should be like: . total 0 crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0 crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0 crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0 crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0 crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0 crw-rw---- 1 root iaa [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#enable-iaa-with-gzip-compression-for-shuffle-compression"
- },"454": {
+ },"455": {
"doc": "Getting Start with Velox Backend",
"title": "IAA references",
"content": "Intel® IAA Enabling Guide . Check out the Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling Guide . Intel® QPL Documentation . Check out the Intel® Query Processing Library (Intel® QPL) Documentation . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#iaa-references",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#iaa-references"
- },"455": {
+ },"456": {
"doc": "Getting Start with Velox Backend",
"title": "Test TPC-H or TPC-DS on Gluten with Velox backend",
"content": "All TPC-H and TPC-DS queries are supported in Gluten Velox backend. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#test-tpc-h-or-tpc-ds-on-gluten-with-velox-backend"
- },"456": {
+ },"457": {
"doc": "Getting Start with Velox Backend",
"title": "Data preparation",
"content": "The data generation scripts are TPC-H dategen script and TPC-DS dategen script. The used TPC-H and TPC-DS queries are the original ones, and can be accessed from TPC-DS queries and TPC-H queries. Some other versions of TPC-DS queries are also provided, but are not recommended for testing, including: . | the modified TPC-DS queries with “Decimal-to-Double”: TPC-DS non-decimal queries (outdated). | . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#data-preparation",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#data-preparation"
- },"457": {
+ },"458": {
"doc": "Getting Start with Velox Backend",
"title": "Submit the Spark SQL job",
"content": "Submit test script from spark-shell. You can find the scala code to Run TPC-H as an example. Please remember to modify the location of TPC-H files as well as TPC-H queries before you run the testing. var parquet_file_path = \"/PATH/TO/TPCH_PARQUET_PATH\" var gluten_root = \"/PATH/TO/GLUTEN\" . Below script shows an example about how to run the testing, you should modify the parameters such as executor cores, memory, offHeap size based on your environment. export GLUTEN_JA [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#submit-the-spark-sql-job",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#submit-the-spark-sql-job"
- },"458": {
+ },"459": {
"doc": "Getting Start with Velox Backend",
"title": "Result",
"content": "wholestagetransformer indicates that the offloading works. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#result",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#result"
- },"459": {
+ },"460": {
"doc": "Getting Start with Velox Backend",
"title": "Performance",
"content": "Below table shows the TPC-H Q1 and Q6 Performance in a multiple-thread test (–num-executors 6 –executor-cores 6) for Velox and vanilla Spark. Both Parquet and ORC datasets are sf1024. | Query Performance (s) | Velox (ORC) | Vanilla Spark (Parquet) | Vanilla Spark (ORC) | . | TPC-H Q6 | 13.6 | 21.6 | 34.9 | . | TPC-H Q1 | 26.1 | 76.7 | 84.9 | . ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#performance",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#performance"
- },"460": {
+ },"461": {
"doc": "Getting Start with Velox Backend",
"title": "External reference setup",
"content": "TO ease your first-hand experience of using Gluten, we have set up an external reference cluster. If you are interested, please contact Weiting.Chen@intel.com. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#external-reference-setup",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#external-reference-setup"
- },"461": {
+ },"462": {
"doc": "Getting Start with Velox Backend",
"title": "Gluten UI",
"content": " ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#gluten-ui",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#gluten-ui"
- },"462": {
+ },"463": {
"doc": "Getting Start with Velox Backend",
"title": "Gluten event",
"content": "Gluten provides two events GlutenBuildInfoEvent and GlutenPlanFallbackEvent: . | GlutenBuildInfoEvent, it contains the Gluten build information so that we are able to be aware of the environment when doing some debug. It includes Java Version, Scala Version, GCC Version, Gluten Version, Spark Version, Hadoop Version, Gluten Revision, Backend, Backend Revision, etc. | GlutenPlanFallbackEvent, it contains the fallback information for each query execution. Note, if the query [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#gluten-event",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#gluten-event"
- },"463": {
+ },"464": {
"doc": "Getting Start with Velox Backend",
"title": "SQL tab",
"content": "Gluten provides a tab based on Spark UI, named Gluten SQL / DataFrame . This tab contains two parts: . | The Gluten build information. | SQL/Dataframe queries fallback information. | . If you want to disable Gluten UI, add a config when submitting --conf spark.gluten.ui.enabled=false. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#sql-tab",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#sql-tab"
- },"464": {
+ },"465": {
"doc": "Getting Start with Velox Backend",
"title": "History server",
"content": "Gluten UI also supports Spark history server. Add gluten-ui jar into the history server classpath, e.g., $SPARK_HOME/jars, then restart history server. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#history-server",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#history-server"
- },"465": {
+ },"466": {
"doc": "Getting Start with Velox Backend",
"title": "Native plan string",
"content": "Gluten supports inject native plan string into Spark explain with formatted mode by setting --conf spark.gluten.sql.injectNativePlanStringToExplain=true. Here is an example, how Gluten show the native plan string. (9) WholeStageCodegenTransformer (2) Input [6]: [c1#0L, c2#1L, c3#2L, c1#3L, c2#4L, c3#5L] Arguments: false Native Plan: -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), (n3_8:BIGINT, \"n0_2\"), (n3_9:BIGINT, \"n1_0\"), (n3_10:BIGINT, \" [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-string",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-string"
- },"466": {
+ },"467": {
"doc": "Getting Start with Velox Backend",
"title": "Native plan with stats",
"content": "Gluten supports print native plan with stats to executor system output stream by setting --conf spark.gluten.sql.debug=true. Note that, the plan string with stats is task level which may cause executor log size big. Here is an example, how Gluten show the native plan string with stats. I20231121 10:19:42.348845 90094332 WholeStageResultIterator.cc:220] Native Plan with stats for: [Stage: 1 TID: 16] -- Project[expressions: (n3_6:BIGINT, \"n0_0\"), (n3_7:BIGINT, \"n0_1\"), [...]
"url": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-with-stats",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#native-plan-with-stats"
- },"467": {
+ },"468": {
"doc": "Getting Start with Velox Backend",
"title": "Gluten Implicits",
"content": "Gluten provides a helper class to get the fallback summary from a Spark Dataset. import org.apache.spark.sql.execution.GlutenImplicits._ val df = spark.sql(\"SELECT * FROM t\") df.fallbackSummary . Note that, if AQE is enabled, but the query is not materialized, then it will re-plan the query execution with disabled AQE. It is a workaround to get the final plan, and it may cause the inconsistent results with a materialized query. However, we have no choice. ",
"url": "/archives/v1.3.0/getting-started/velox-backend/#gluten-implicits",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/#gluten-implicits"
- },"468": {
+ },"469": {
"doc": "Getting Start with Velox Backend",
"title": "Getting Start with Velox Backend",
"content": " ",
"url": "/archives/v1.3.0/getting-started/velox-backend/",
"relUrl": "/archives/v1.3.0/getting-started/velox-backend/"
- },"469": {
+ },"470": {
"doc": "Using ABFS with Gluten",
"title": "Working with ABFS",
"content": " ",
"url": "/archives/v1.3.0/getting-started/abfs/#working-with-abfs",
"relUrl": "/archives/v1.3.0/getting-started/abfs/#working-with-abfs"
- },"470": {
+ },"471": {
"doc": "Using ABFS with Gluten",
"title": "Configuring ABFS Access Token",
"content": "To configure access to your storage account, replace with the name of your account. This property aligns with Spark configurations. By setting this config multiple times using different storage account names, you can access multiple ABFS accounts. spark.hadoop.fs.azure.account.key.<storage-account>.dfs.core.windows.net XXXXXXXXX . Other authentatication methods are not yet supported. ",
"url": "/archives/v1.3.0/getting-started/abfs/#configuring-abfs-access-token",
"relUrl": "/archives/v1.3.0/getting-started/abfs/#configuring-abfs-access-token"
- },"471": {
+ },"472": {
"doc": "Using ABFS with Gluten",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from ABFS. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/archives/v1.3.0/getting-started/abfs/#local-caching-support",
"relUrl": "/archives/v1.3.0/getting-started/abfs/#local-caching-support"
- },"472": {
+ },"473": {
"doc": "Using ABFS with Gluten",
"title": "Using ABFS with Gluten",
"content": "ABFS is an important data store for big data users. This doc discusses config details and use cases of Gluten with ABFS. To use an ABFS account as your data source, please ensure you use the listed ABFS config in your spark-defaults.conf. If you would like to authenticate with ABFS using additional auth mechanisms, please reach out using the ‘Issues’ tab. ",
"url": "/archives/v1.3.0/getting-started/abfs/",
"relUrl": "/archives/v1.3.0/getting-started/abfs/"
- },"473": {
+ },"474": {
"doc": "Using GCS with Gluten",
"title": "Working with GCS",
"content": " ",
"url": "/archives/v1.3.0/getting-started/gcs/#working-with-gcs",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#working-with-gcs"
- },"474": {
+ },"475": {
"doc": "Using GCS with Gluten",
"title": "Installing the gcloud CLI",
"content": "To access GCS Objects using Gluten and Velox, first you have to [download an install the gcloud CLI] (https://cloud.google.com/sdk/docs/install). ",
"url": "/archives/v1.3.0/getting-started/gcs/#installing-the-gcloud-cli",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#installing-the-gcloud-cli"
- },"475": {
+ },"476": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS using a user account",
"content": "This is recommended for regular users, follow the instructions to authorize a user account. After these steps, no specific configuration is required for Gluten, since the authorization was handled entirely by the gcloud tool. ",
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-user-account",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-user-account"
- },"476": {
+ },"477": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS using a credential file",
"content": "For workloads that need to be fully automated, manually authorizing can be problematic. For such cases it is better to use a json file with the credentials. This is described in the [instructions to configure a service account]https://cloud.google.com/sdk/docs/authorizing#service-account. Such json file with the credetials can be passed to Gluten: . spark.hadoop.fs.gs.auth.type SERVICE_ACCOUNT_JSON_KEYFILE spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to th [...]
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-credential-file",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-using-a-credential-file"
- },"477": {
+ },"478": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS endpoints",
"content": "For cases when a GCS mock is used, an optional endpoint can be provided: . spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https . ",
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-endpoints",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-endpoints"
- },"478": {
+ },"479": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS max retry count",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected . ",
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-count",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-count"
- },"479": {
+ },"480": {
"doc": "Using GCS with Gluten",
"title": "Configuring GCS max retry time",
"content": "For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retring (10s, 1m, etc). ",
"url": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-time",
"relUrl": "/archives/v1.3.0/getting-started/gcs/#configuring-gcs-max-retry-time"
- },"480": {
+ },"481": {
"doc": "Using GCS with Gluten",
"title": "Using GCS with Gluten",
"content": "Object stores offered by CSPs such as GCS are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use a GCS endpoint as your data source, please ensure you are using the following GCS configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to GCS with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.3.0/getting-started/gcs/",
"relUrl": "/archives/v1.3.0/getting-started/gcs/"
- },"481": {
+ },"482": {
"doc": "Velox Local Caching",
"title": "Velox Local Caching",
"content": "Velox supports a local cache when reading data from HDFS/S3/ABFS. With this feature, Velox can asynchronously cache the data on local disk when reading from remote storage and future read requests on previously cached blocks will be serviced from local cache files. To enable the local caching feature, the following configurations are required: . spark.gluten.sql.columnar.backend.velox.cacheEnabled // enable or disable velox cache, default false. spark.gluten.sql.columnar. [...]
"url": "/archives/v1.3.0/getting-started/localcache/",
"relUrl": "/archives/v1.3.0/getting-started/localcache/"
- },"482": {
+ },"483": {
"doc": "Using S3 with Gluten",
"title": "Working with S3",
"content": " ",
"url": "/archives/v1.3.0/getting-started/s3/#working-with-s3",
"relUrl": "/archives/v1.3.0/getting-started/s3/#working-with-s3"
- },"483": {
+ },"484": {
"doc": "Using S3 with Gluten",
"title": "Configuring S3 endpoint",
"content": "S3 provides the endpoint based method to access the files, here’s the example configuration. Users may need to modify some values based on real setup. spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider spark.hadoop.fs.s3a.access.key XXXXXXXXX spark.hadoop.fs.s3a.secret.key XXXXXXXXX spark.hadoop.fs.s3a.endpoint https://s3.us-west-1.amazonaws.com spark.hadoop.fs. [...]
"url": "/archives/v1.3.0/getting-started/s3/#configuring-s3-endpoint",
"relUrl": "/archives/v1.3.0/getting-started/s3/#configuring-s3-endpoint"
- },"484": {
+ },"485": {
"doc": "Using S3 with Gluten",
"title": "Configuring S3 instance credentials",
"content": "S3 also provides other methods for accessing, you can also use instance credentials by setting the following config . spark.hadoop.fs.s3a.use.instance.credentials true . Note that in this case, “spark.hadoop.fs.s3a.endpoint” won’t take affect as Gluten will use the endpoint set during instance creation. ",
"url": "/archives/v1.3.0/getting-started/s3/#configuring-s3-instance-credentials",
"relUrl": "/archives/v1.3.0/getting-started/s3/#configuring-s3-instance-credentials"
- },"485": {
+ },"486": {
"doc": "Using S3 with Gluten",
"title": "Configuring S3 IAM roles",
"content": "You can also use iam role credentials by setting the following configurations. Instance credentials have higher priority than iam credentials. spark.hadoop.fs.s3a.iam.role xxxx spark.hadoop.fs.s3a.iam.role.session.name xxxx . Note that spark.hadoop.fs.s3a.iam.role.session.name is optional. ",
"url": "/archives/v1.3.0/getting-started/s3/#configuring-s3-iam-roles",
"relUrl": "/archives/v1.3.0/getting-started/s3/#configuring-s3-iam-roles"
- },"486": {
+ },"487": {
"doc": "Using S3 with Gluten",
"title": "Other authentatication methods are not supported yet",
"content": ". | AWS temporary credential | . ",
"url": "/archives/v1.3.0/getting-started/s3/#other-authentatication-methods-are-not-supported-yet",
"relUrl": "/archives/v1.3.0/getting-started/s3/#other-authentatication-methods-are-not-supported-yet"
- },"487": {
+ },"488": {
"doc": "Using S3 with Gluten",
"title": "Log granularity of AWS C++ SDK in velox",
"content": "You can change log granularity of AWS C++ SDK by setting the spark.gluten.velox.awsSdkLogLevel configuration. The Allowed values are: . | OFF | FATAL | ERROR | WARN | INFO | DEBUG | TRACE | . ",
"url": "/archives/v1.3.0/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox",
"relUrl": "/archives/v1.3.0/getting-started/s3/#log-granularity-of-aws-c-sdk-in-velox"
- },"488": {
+ },"489": {
"doc": "Using S3 with Gluten",
"title": "Local Caching support",
"content": "Velox supports a local cache when reading data from S3. Please refer Velox Local Cache part for more detailed configurations. ",
"url": "/archives/v1.3.0/getting-started/s3/#local-caching-support",
"relUrl": "/archives/v1.3.0/getting-started/s3/#local-caching-support"
- },"489": {
+ },"490": {
"doc": "Using S3 with Gluten",
"title": "Using S3 with Gluten",
"content": "Object stores offered by CSPs such as AWS S3 are important for users of Gluten to store their data. This doc will discuss all details of configs, and use cases around using Gluten with object stores. In order to use an S3 endpoint as your data source, please ensure you are using the following S3 configs in your spark-defaults.conf. If you’re experiencing any issues authenticating to S3 with additional auth mechanisms, please reach out to us using the ‘Issues’ tab. ",
"url": "/archives/v1.3.0/getting-started/s3/",
"relUrl": "/archives/v1.3.0/getting-started/s3/"
- },"490": {
+ },"491": {
"doc": "Build Parameters for Velox Backend",
"title": "Build Parameters",
"content": "Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh . Please set them via --, e.g. --build_type=Release. | Parameters | Description | Default | . | build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | . | build_tests | Build gluten cpp tests. | OFF | . | build_examples | Build udf example. | OFF | . | build_benchmarks | Build gluten cpp benchmarks. | OFF | . | build_jemalloc | Build with jemalloc. | OFF | . | build_proto [...]
"url": "/archives/v1.3.0/getting-started/build-guide/#build-parameters",
"relUrl": "/archives/v1.3.0/getting-started/build-guide/#build-parameters"
- },"491": {
+ },"492": {
"doc": "Build Parameters for Velox Backend",
"title": "Gluten Jar for Deployment",
"content": "The gluten jar built out is under GLUTEN_SRC/package/target/. It’s name pattern is gluten-<backend_type>-bundle-spark<spark.bundle.version>_<scala.binary.version>-<os.detected.release>_<os.detected.release.version>-<project.version>.jar. | Spark Version | spark.bundle.version | scala.binary.version | . | 3.2.2 | 3.2 | 2.12 | . | 3.3.1 | 3.3 | 2.12 | . | 3.4.2 | 3.4 | 2.12 | . | 3.5.1 | 3.5 | 2.12 | . ",
"url": "/archives/v1.3.0/getting-started/build-guide/#gluten-jar-for-deployment",
"relUrl": "/archives/v1.3.0/getting-started/build-guide/#gluten-jar-for-deployment"
- },"492": {
+ },"493": {
"doc": "Build Parameters for Velox Backend",
"title": "Build Parameters for Velox Backend",
"content": " ",
"url": "/archives/v1.3.0/getting-started/build-guide/",
"relUrl": "/archives/v1.3.0/getting-started/build-guide/"
- },"493": {
+ },"494": {
"doc": "Getting-Started",
"title": "Getting Started with Gluten for Apache Spark",
"content": " ",
"url": "/archives/v1.3.0/getting-started#getting-started-with-gluten-for-apache-spark",
"relUrl": "/archives/v1.3.0/getting-started#getting-started-with-gluten-for-apache-spark"
- },"494": {
+ },"495": {
"doc": "Getting-Started",
"title": "Getting-Started",
"content": " ",
"url": "/archives/v1.3.0/getting-started",
"relUrl": "/archives/v1.3.0/getting-started"
- },"495": {
+ },"496": {
"doc": "v1.3.0",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.3.0/#gluten-documents-by-version",
"relUrl": "/archives/v1.3.0/#gluten-documents-by-version"
- },"496": {
+ },"497": {
"doc": "v1.3.0",
"title": "v1.3.0",
"content": " ",
"url": "/archives/v1.3.0/",
"relUrl": "/archives/v1.3.0/"
- },"497": {
+ },"498": {
"doc": "Velox Backend",
"title": "Gluten Documents by version",
"content": " ",
"url": "/archives/v1.3.0/velox-backend#gluten-documents-by-version",
"relUrl": "/archives/v1.3.0/velox-backend#gluten-documents-by-version"
- },"498": {
+ },"499": {
"doc": "Velox Backend",
"title": "Velox Backend",
"content": " ",
"url": "/archives/v1.3.0/velox-backend",
"relUrl": "/archives/v1.3.0/velox-backend"
- },"499": {
+ },"500": {
"doc": "Limitations",
"title": "Velox Backend Limitations",
"content": "This document describes the limitations of velox backend by listing some known cases where exception will be thrown, gluten behaves incompatibly with spark, or certain plan’s execution must fall back to vanilla spark, etc. Override of Spark classes (For Spark3.2 and Spark3.3) . Gluten avoids to modify Spark’s existing code and use Spark APIs if possible. However, some APIs aren’t exposed in Vanilla spark and we have to copy the Spark file and do the hardcode changes. The [...]
"url": "/archives/v1.3.0/velox-backend/limitations/#velox-backend-limitations",
"relUrl": "/archives/v1.3.0/velox-backend/limitations/#velox-backend-limitations"
- },"500": {
+ },"501": {
"doc": "Limitations",
"title": "Limitations",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/limitations/",
"relUrl": "/archives/v1.3.0/velox-backend/limitations/"
- },"501": {
+ },"502": {
"doc": "Supported Operators & Functions",
"title": "The Operators and Functions Support Progress",
"content": "Gluten is still under active development. Here is a list of supported operators and functions. Since the same function may have different semantics between Presto and Spark, Velox implement the functions in Presto category, if we note a different semantics from Spark, then the function is implemented in Spark category. So Gluten will first try to find function in Velox’s spark category, if a function isn’t implemented then refer to Presto category. The total number of fun [...]
"url": "/archives/v1.3.0/velox-backend/support/#the-operators-and-functions-support-progress",
"relUrl": "/archives/v1.3.0/velox-backend/support/#the-operators-and-functions-support-progress"
- },"502": {
+ },"503": {
"doc": "Supported Operators & Functions",
"title": "Supported Operators & Functions",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/support/",
"relUrl": "/archives/v1.3.0/velox-backend/support/"
- },"503": {
+ },"504": {
"doc": "Troubleshooting",
"title": "Velox Backend Troubleshooting",
"content": "Fatal error after native exception is thrown . We depend on checking exceptions thrown from native code to validate whether a spark plan can be really offloaded to native engine. But if libunwind-dev is installed, native exception will not be caught and will interrupt the program. So far, we observed this fatal error can happen only on Ubuntu 20.04. Please remove libunwind-dev and then re-build the project to address this issue. sudo apt-get purge --auto-remove libunwind- [...]
"url": "/archives/v1.3.0/velox-backend/troubleshooting/#velox-backend-troubleshooting",
"relUrl": "/archives/v1.3.0/velox-backend/troubleshooting/#velox-backend-troubleshooting"
- },"504": {
+ },"505": {
"doc": "Troubleshooting",
"title": "Troubleshooting",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/troubleshooting/",
"relUrl": "/archives/v1.3.0/velox-backend/troubleshooting/"
- },"505": {
+ },"506": {
"doc": "Velox UDF",
"title": "Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF)",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#velox-user-defined-functions-udf-and-user-defined-aggregate-functions-udaf"
- },"506": {
+ },"507": {
"doc": "Velox UDF",
"title": "Introduction",
"content": "Velox backend supports User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF). Users can create their own functions using the UDF interface provided in Velox backend and build libraries for these functions. At runtime, the UDF are registered at the start of applications. Once registered, Gluten will be able to parse and offload these UDF into Velox during execution. ",
"url": "/archives/v1.3.0/velox-backend/udf/#introduction",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#introduction"
- },"507": {
+ },"508": {
"doc": "Velox UDF",
"title": "Create and Build UDF/UDAF library",
"content": "The following steps demonstrate how to set up a UDF library project: . | Include the UDF Interface Header: First, include the UDF interface header file Udf.h in the project file. The header file defines the UdfEntry struct, along with the macros for declaring the necessary functions to integrate the UDF into Gluten and Velox. | Implement the UDF: Implement UDF. These functions should be able to register to Velox. | Implement the Interface Functions: Implement the followin [...]
"url": "/archives/v1.3.0/velox-backend/udf/#create-and-build-udfudaf-library",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#create-and-build-udfudaf-library"
- },"508": {
+ },"509": {
"doc": "Velox UDF",
"title": "Using UDF/UDAF in Gluten",
"content": "Gluten loads the UDF libraries at runtime. You can upload UDF libraries via --files or --archives, and configure the library paths using the provided Spark configuration, which accepts comma separated list of library paths. Note if running on Yarn client mode, the uploaded files are not reachable on driver side. Users should copy those files to somewhere reachable for driver and set spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths. This configuration is also [...]
"url": "/archives/v1.3.0/velox-backend/udf/#using-udfudaf-in-gluten",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#using-udfudaf-in-gluten"
- },"509": {
+ },"510": {
"doc": "Velox UDF",
"title": "Try the example",
"content": "We provided Velox UDF examples in file MyUDF.cc and UDAF examples in file MyUDAF.cc. You need to build the gluten project with --build_example=ON to get the example libraries./dev/buildbundle-veloxbe.sh --build_examples=ON . Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ . Start spark-shell or spark-sql with below configuration . # Use the `--files` option to upload a library and configure its relative path --files /path/to/glute [...]
"url": "/archives/v1.3.0/velox-backend/udf/#try-the-example",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#try-the-example"
- },"510": {
+ },"511": {
"doc": "Velox UDF",
"title": "Configurations",
"content": "| Parameters | Description | . | spark.gluten.sql.columnar.backend.velox.udfLibraryPaths | Path to the udf/udaf libraries. | . | spark.gluten.sql.columnar.backend.velox.driver.udfLibraryPaths | Path to the udf/udaf libraries on driver node. Only applicable on yarn-client mode. | . | spark.gluten.sql.columnar.backend.velox.udfAllowTypeConversion | Whether to inject possible cast to convert mismatched data types from input to one registered signatures. | . ",
"url": "/archives/v1.3.0/velox-backend/udf/#configurations",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#configurations"
- },"511": {
+ },"512": {
"doc": "Velox UDF",
"title": "Pandas UDFs (a.k.a. Vectorized UDFs)",
"content": " ",
"url": "/archives/v1.3.0/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#pandas-udfs-aka-vectorized-udfs"
- },"512": {
+ },"513": {
"doc": "Velox UDF",
"title": "Introduction",
"content": "Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer doc. ",
"url": "/archives/v1.3.0/velox-backend/udf/#introduction-1",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#introduction-1"
- },"513": {
+ },"514": {
"doc": "Velox UDF",
"title": "Using Pandas UDFs in Gluten with Velox Backend",
"content": "Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: . pip3 install pyspark==$SPARK_VERSION cython pip3 install pandas pyarrow . Gluten provides a config to control enable ColumnarArrowEvalPython or not, with true as defalt. spark.gluten.sql.columnar.arrowUdf . Then take following PySpark code for example: . from pyspark.sql.functions import pandas_udf, PandasUDFType import pyspark.sql.functions as [...]
"url": "/archives/v1.3.0/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend",
"relUrl": "/archives/v1.3.0/velox-backend/udf/#using-pandas-udfs-in-gluten-with-velox-backend"
- },"514": {
+ },"515": {
"doc": "Velox UDF",
"title": "Velox UDF",
"content": " ",
diff --git a/output/docs/developers/how-to-release/index.html b/output/docs/developers/how-to-release/index.html
index 008b146..d750fb0 100644
--- a/output/docs/developers/how-to-release/index.html
+++ b/output/docs/developers/how-to-release/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } .site-nav > ul.nav-category-list > li > button svg { transform: rotate(-90deg); } .site-nav > ul.nav-category-list > li.nav-li [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav ul li a { background-image: none; } .site-nav > ul.nav-category-list > li > button svg { transform: rotate(-90deg); } .site-nav > ul.nav-category-list > li.nav-li [...]
<span class="nv">$ </span>gpg <span class="nt">--full-generate-key</span>
<span class="c"># list the GPG keys</span>
diff --git a/output/feed.xml b/output/feed.xml
index 2393bdb..a780206 100644
--- a/output/feed.xml
+++ b/output/feed.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="3.9.3">Jekyll</generator><link href="https://gluten.apache.org/feed.xml" rel="self" type="application/atom+xml" /><link href="https://gluten.apache.org/" rel="alternate" type="text/html" /><updated>2026-03-10T12:44:30+00:00</updated><id>https://gluten.apache.org/feed.xml</id><title type="html">Apache Gluten incubating</title><subtitle>Gluten is a middle layer r [...]
+<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="3.9.3">Jekyll</generator><link href="https://gluten.apache.org/feed.xml" rel="self" type="application/atom+xml" /><link href="https://gluten.apache.org/" rel="alternate" type="text/html" /><updated>2026-03-16T07:26:25+00:00</updated><id>https://gluten.apache.org/feed.xml</id><title type="html">Apache Gluten incubating</title><subtitle>Gluten is a middle layer r [...]
<img width="800" alt="image" src="/assets/images/GlutenCon2025/poster.png" />
</div></content><author><name></name></author><summary type="html"></summary></entry><entry><title type="html">Glutencon2025 Chinese</title><link href="https://gluten.apache.org/2025/11/06/GlutenCon2025-Chinese.html" rel="alternate" type="text/html" title="Glutencon2025 Chinese" /><published>2025-11-06T00:00:00+00:00</published><updated>2025-11-06T00:00:00+00:00</updated><id>https://gluten.apache.org/2025/11/06/GlutenCon2025-Chinese</id><content type="html" xml:base="https://gluten [...]
diff --git a/output/references/index.html b/output/references/index.html
index fb12d15..eef4095 100644
--- a/output/references/index.html
+++ b/output/references/index.html
@@ -1 +1 @@
-<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(4)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
+<!DOCTYPE html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"> <link rel="stylesheet" href="/assets/css/just-the-docs-default.css"> <link rel="stylesheet" href="/assets/css/just-the-docs-head-nav.css" id="jtd-head-nav-stylesheet"> <style id="jtd-nav-activation"> .site-nav > ul.nav-list:first-child > li:not(:nth-child(4)) > a, .site-nav > ul.nav-list:first-child > li > ul > li a { background-image: none; } .site-nav > ul.nav-list:no [...]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org