You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by gi...@apache.org on 2022/04/29 10:44:24 UTC
[incubator-kyuubi-website] branch asf-site updated: deploy: 88b5a93c24b5aaa14802e2f29734863c99a6de9e
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi-website.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 1c0b8d9 deploy: 88b5a93c24b5aaa14802e2f29734863c99a6de9e
1c0b8d9 is described below
commit 1c0b8d9daee36d647853c2d4b31d754acc2d9a93
Author: turboFei <tu...@users.noreply.github.com>
AuthorDate: Fri Apr 29 10:43:58 2022 +0000
deploy: 88b5a93c24b5aaa14802e2f29734863c99a6de9e
---
content/docs/latest/.buildinfo | 2 +-
content/docs/latest/404.html | 2 +-
.../docs/latest/_static/documentation_options.js | 2 +-
content/docs/latest/appendix/index.html | 2 +-
content/docs/latest/appendix/terminology.html | 2 +-
content/docs/latest/client/hive_jdbc.html | 2 +-
content/docs/latest/client/index.html | 2 +-
content/docs/latest/client/kerberized_kyuubi.html | 2 +-
content/docs/latest/community/CONTRIBUTING.html | 2 +-
content/docs/latest/community/collaborators.html | 2 +-
content/docs/latest/community/index.html | 2 +-
content/docs/latest/community/release.html | 2 +-
.../docs/latest/deployment/engine_lifecycle.html | 8 +-
.../latest/deployment/engine_on_kubernetes.html | 4 +-
content/docs/latest/deployment/engine_on_yarn.html | 4 +-
.../docs/latest/deployment/engine_share_level.html | 4 +-
.../latest/deployment/high_availability_guide.html | 2 +-
content/docs/latest/deployment/hive_metastore.html | 2 +-
content/docs/latest/deployment/index.html | 12 +-
.../latest/deployment/kyuubi_on_kubernetes.html | 2 +-
content/docs/latest/deployment/settings.html | 6 +-
content/docs/latest/deployment/spark/aqe.html | 56 +++----
content/docs/latest/deployment/spark/basics.html | 28 ++--
content/docs/latest/deployment/spark/driver.html | 22 +--
.../deployment/spark/dynamic_allocation.html | 94 +++++------
.../deployment/spark/dynamicpartitionpruning.html | 10 +-
content/docs/latest/deployment/spark/ess.html | 10 +-
.../docs/latest/deployment/spark/eventqueue.html | 14 +-
content/docs/latest/deployment/spark/executor.html | 14 +-
.../docs/latest/deployment/spark/heartbeart.html | 8 +-
.../deployment/spark/incremental_collection.html | 26 +--
content/docs/latest/deployment/spark/index.html | 50 +++---
content/docs/latest/deployment/spark/locality.html | 12 +-
.../docs/latest/deployment/spark/monitering.html | 16 +-
content/docs/latest/deployment/spark/shuffle.html | 56 +++----
.../docs/latest/deployment/spark/speculation.html | 14 +-
content/docs/latest/deployment/spark/sql.html | 24 +--
.../docs/latest/develop_tools/build_document.html | 2 +-
content/docs/latest/develop_tools/building.html | 2 +-
content/docs/latest/develop_tools/debugging.html | 2 +-
content/docs/latest/develop_tools/developer.html | 2 +-
.../docs/latest/develop_tools/distribution.html | 2 +-
content/docs/latest/develop_tools/index.html | 2 +-
content/docs/latest/develop_tools/testing.html | 2 +-
content/docs/latest/genindex.html | 2 +-
content/docs/latest/index.html | 2 +-
content/docs/latest/integrations/delta_lake.html | 2 +-
.../integrations/delta_lake_with_azure_blob.html | 60 +++----
content/docs/latest/integrations/index.html | 2 +-
content/docs/latest/integrations/kudu.html | 174 ++++++++++-----------
content/docs/latest/monitor/events.html | 2 +-
content/docs/latest/monitor/index.html | 2 +-
content/docs/latest/monitor/logging.html | 16 +-
content/docs/latest/monitor/metrics.html | 2 +-
content/docs/latest/monitor/trouble_shooting.html | 2 +-
content/docs/latest/overview/architecture.html | 2 +-
content/docs/latest/overview/index.html | 2 +-
content/docs/latest/overview/kyuubi_vs_hive.html | 2 +-
.../latest/overview/kyuubi_vs_thriftserver.html | 2 +-
content/docs/latest/quick_start/index.html | 2 +-
content/docs/latest/quick_start/quick_start.html | 2 +-
.../quick_start/quick_start_with_beeline.html | 2 +-
.../quick_start/quick_start_with_datagrip.html | 2 +-
.../quick_start/quick_start_with_dbeaver.html | 110 ++++++-------
.../latest/quick_start/quick_start_with_helm.html | 50 +++---
.../latest/quick_start/quick_start_with_hue.html | 46 +++---
.../latest/quick_start/quick_start_with_jdbc.html | 6 +-
.../quick_start/quick_start_with_jupyter.html | 2 +-
content/docs/latest/search.html | 2 +-
content/docs/latest/searchindex.js | 2 +-
content/docs/latest/security/authentication.html | 22 +--
content/docs/latest/security/authorization.html | 2 +-
.../security/hadoop_credentials_manager.html | 52 +++++-
content/docs/latest/security/index.html | 6 +-
content/docs/latest/security/kinit.html | 10 +-
content/docs/latest/sql/functions.html | 2 +-
content/docs/latest/sql/index.html | 2 +-
content/docs/latest/sql/rules.html | 2 +-
content/docs/latest/sql/z-order-benchmark.html | 16 +-
content/docs/latest/sql/z-order-introduction.html | 18 +--
content/docs/latest/tools/index.html | 2 +-
content/docs/latest/tools/spark_block_cleaner.html | 74 ++++-----
82 files changed, 642 insertions(+), 602 deletions(-)
diff --git a/content/docs/latest/.buildinfo b/content/docs/latest/.buildinfo
index 876ddd5..d02a3cf 100644
--- a/content/docs/latest/.buildinfo
+++ b/content/docs/latest/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 795b428079ed6c040d3b359f85cf40e1
+config: 54979cdc96d7fec1e4892cfc1eb9bf2a
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/content/docs/latest/404.html b/content/docs/latest/404.html
index 40e1676..30c6da7 100644
--- a/content/docs/latest/404.html
+++ b/content/docs/latest/404.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Page not found — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Page not found — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/_static/documentation_options.js b/content/docs/latest/_static/documentation_options.js
index 87d1a8c..55eff3c 100644
--- a/content/docs/latest/_static/documentation_options.js
+++ b/content/docs/latest/_static/documentation_options.js
@@ -1,6 +1,6 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
- VERSION: '1.5.0-incubating',
+ VERSION: '1.5.1-incubating',
LANGUAGE: 'None',
COLLAPSE_INDEX: false,
BUILDER: 'html',
diff --git a/content/docs/latest/appendix/index.html b/content/docs/latest/appendix/index.html
index fcbf458..84ac995 100644
--- a/content/docs/latest/appendix/index.html
+++ b/content/docs/latest/appendix/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Appendixes — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Appendixes — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/appendix/terminology.html b/content/docs/latest/appendix/terminology.html
index 3f936cf..f765b19 100644
--- a/content/docs/latest/appendix/terminology.html
+++ b/content/docs/latest/appendix/terminology.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Terminologies — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Terminologies — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/client/hive_jdbc.html b/content/docs/latest/client/hive_jdbc.html
index 77303d7..6a47e22 100644
--- a/content/docs/latest/client/hive_jdbc.html
+++ b/content/docs/latest/client/hive_jdbc.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Access Kyuubi with Hive JDBC and ODBC Drivers — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Access Kyuubi with Hive JDBC and ODBC Drivers — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/client/index.html b/content/docs/latest/client/index.html
index 674e290..ada7ab6 100644
--- a/content/docs/latest/client/index.html
+++ b/content/docs/latest/client/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Client Documentation — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Client Documentation — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/client/kerberized_kyuubi.html b/content/docs/latest/client/kerberized_kyuubi.html
index a168d93..4e0c83b 100644
--- a/content/docs/latest/client/kerberized_kyuubi.html
+++ b/content/docs/latest/client/kerberized_kyuubi.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Access Kerberized Kyuubi with Beeline & BI Tools — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Access Kerberized Kyuubi with Beeline & BI Tools — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/community/CONTRIBUTING.html b/content/docs/latest/community/CONTRIBUTING.html
index 25022a7..3d739b2 100644
--- a/content/docs/latest/community/CONTRIBUTING.html
+++ b/content/docs/latest/community/CONTRIBUTING.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Contributing to Apache Kyuubi — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Contributing to Apache Kyuubi — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/community/collaborators.html b/content/docs/latest/community/collaborators.html
index 4fcb99b..47428f9 100644
--- a/content/docs/latest/community/collaborators.html
+++ b/content/docs/latest/community/collaborators.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Collaborators — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Collaborators — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/community/index.html b/content/docs/latest/community/index.html
index 131fa32..58c21fd 100644
--- a/content/docs/latest/community/index.html
+++ b/content/docs/latest/community/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Community — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Community — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/community/release.html b/content/docs/latest/community/release.html
index 7db6ab6..7edd7a6 100644
--- a/content/docs/latest/community/release.html
+++ b/content/docs/latest/community/release.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. Kyuubi Release Guide — Kyuubi 1.5.0-incubating documentation</title>
+ <title>3. Kyuubi Release Guide — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/deployment/engine_lifecycle.html b/content/docs/latest/deployment/engine_lifecycle.html
index 38e47f5..e2f46ad 100644
--- a/content/docs/latest/deployment/engine_lifecycle.html
+++ b/content/docs/latest/deployment/engine_lifecycle.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>4. The TTL Of Kyuubi Engines — Kyuubi 1.5.0-incubating documentation</title>
+ <title>4. The TTL Of Kyuubi Engines — Kyuubi 1.5.1-incubating documentation</title>
@@ -41,7 +41,7 @@
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
- <link rel="next" title="The Spark SQL Engine Configuration Guide" href="spark/index.html" />
+ <link rel="next" title="5. The Spark SQL Engine Configuration Guide" href="spark/index.html" />
<link rel="prev" title="3. The Share Level Of Kyuubi Engines" href="engine_share_level.html" />
</head>
@@ -106,7 +106,7 @@
<li class="toctree-l4"><a class="reference internal" href="#configurations">4.3. Configurations</a></li>
</ul>
</li>
-<li class="toctree-l3"><a class="reference internal" href="spark/index.html">The Spark SQL Engine Configuration Guide</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark/index.html">5. The Spark SQL Engine Configuration Guide</a></li>
</ul>
</li>
</ul>
@@ -302,7 +302,7 @@ Note that, for <a class="reference external" href="engine_share_level.html#conne
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
- <a href="spark/index.html" class="btn btn-neutral float-right" title="The Spark SQL Engine Configuration Guide" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+ <a href="spark/index.html" class="btn btn-neutral float-right" title="5. The Spark SQL Engine Configuration Guide" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
<a href="engine_share_level.html" class="btn btn-neutral float-left" title="3. The Share Level Of Kyuubi Engines" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
</div>
diff --git a/content/docs/latest/deployment/engine_on_kubernetes.html b/content/docs/latest/deployment/engine_on_kubernetes.html
index ce09f91..6742981 100644
--- a/content/docs/latest/deployment/engine_on_kubernetes.html
+++ b/content/docs/latest/deployment/engine_on_kubernetes.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Deploy Kyuubi engines on Kubernetes — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Deploy Kyuubi engines on Kubernetes — Kyuubi 1.5.1-incubating documentation</title>
@@ -105,7 +105,7 @@
</li>
<li class="toctree-l3"><a class="reference internal" href="engine_share_level.html">3. The Share Level Of Kyuubi Engines</a></li>
<li class="toctree-l3"><a class="reference internal" href="engine_lifecycle.html">4. The TTL Of Kyuubi Engines</a></li>
-<li class="toctree-l3"><a class="reference internal" href="spark/index.html">The Spark SQL Engine Configuration Guide</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark/index.html">5. The Spark SQL Engine Configuration Guide</a></li>
</ul>
</li>
</ul>
diff --git a/content/docs/latest/deployment/engine_on_yarn.html b/content/docs/latest/deployment/engine_on_yarn.html
index 5d19e18..96ad0a9 100644
--- a/content/docs/latest/deployment/engine_on_yarn.html
+++ b/content/docs/latest/deployment/engine_on_yarn.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Deploy Kyuubi engines on Yarn — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Deploy Kyuubi engines on Yarn — Kyuubi 1.5.1-incubating documentation</title>
@@ -106,7 +106,7 @@
<li class="toctree-l3"><a class="reference internal" href="engine_on_kubernetes.html">2. Deploy Kyuubi engines on Kubernetes</a></li>
<li class="toctree-l3"><a class="reference internal" href="engine_share_level.html">3. The Share Level Of Kyuubi Engines</a></li>
<li class="toctree-l3"><a class="reference internal" href="engine_lifecycle.html">4. The TTL Of Kyuubi Engines</a></li>
-<li class="toctree-l3"><a class="reference internal" href="spark/index.html">The Spark SQL Engine Configuration Guide</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark/index.html">5. The Spark SQL Engine Configuration Guide</a></li>
</ul>
</li>
</ul>
diff --git a/content/docs/latest/deployment/engine_share_level.html b/content/docs/latest/deployment/engine_share_level.html
index d39939b..77e426d 100644
--- a/content/docs/latest/deployment/engine_share_level.html
+++ b/content/docs/latest/deployment/engine_share_level.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. The Share Level Of Kyuubi Engines — Kyuubi 1.5.0-incubating documentation</title>
+ <title>3. The Share Level Of Kyuubi Engines — Kyuubi 1.5.1-incubating documentation</title>
@@ -107,7 +107,7 @@
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="engine_lifecycle.html">4. The TTL Of Kyuubi Engines</a></li>
-<li class="toctree-l3"><a class="reference internal" href="spark/index.html">The Spark SQL Engine Configuration Guide</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark/index.html">5. The Spark SQL Engine Configuration Guide</a></li>
</ul>
</li>
</ul>
diff --git a/content/docs/latest/deployment/high_availability_guide.html b/content/docs/latest/deployment/high_availability_guide.html
index f2b1898..ea2ff06 100644
--- a/content/docs/latest/deployment/high_availability_guide.html
+++ b/content/docs/latest/deployment/high_availability_guide.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. Kyuubi High Availability Guide — Kyuubi 1.5.0-incubating documentation</title>
+ <title>3. Kyuubi High Availability Guide — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/deployment/hive_metastore.html b/content/docs/latest/deployment/hive_metastore.html
index 85dd739..7edc2c0 100644
--- a/content/docs/latest/deployment/hive_metastore.html
+++ b/content/docs/latest/deployment/hive_metastore.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Integration with Hive Metastore — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Integration with Hive Metastore — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/deployment/index.html b/content/docs/latest/deployment/index.html
index 003a1c3..f9c8eb0 100644
--- a/content/docs/latest/deployment/index.html
+++ b/content/docs/latest/deployment/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Deploying Kyuubi — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Deploying Kyuubi — Kyuubi 1.5.1-incubating documentation</title>
@@ -109,7 +109,7 @@
<li class="toctree-l3"><a class="reference internal" href="engine_on_kubernetes.html">2. Deploy Kyuubi engines on Kubernetes</a></li>
<li class="toctree-l3"><a class="reference internal" href="engine_share_level.html">3. The Share Level Of Kyuubi Engines</a></li>
<li class="toctree-l3"><a class="reference internal" href="engine_lifecycle.html">4. The TTL Of Kyuubi Engines</a></li>
-<li class="toctree-l3"><a class="reference internal" href="spark/index.html">The Spark SQL Engine Configuration Guide</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark/index.html">5. The Spark SQL Engine Configuration Guide</a></li>
</ul>
</li>
</ul>
@@ -280,10 +280,10 @@
<li class="toctree-l2"><a class="reference internal" href="engine_lifecycle.html#configurations">4.3. Configurations</a></li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="spark/index.html">The Spark SQL Engine Configuration Guide</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="spark/dynamic_allocation.html">1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
-<li class="toctree-l2"><a class="reference internal" href="spark/aqe.html">2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
-<li class="toctree-l2"><a class="reference internal" href="spark/incremental_collection.html">3. Solution for Big Result Sets</a></li>
+<li class="toctree-l1"><a class="reference internal" href="spark/index.html">5. The Spark SQL Engine Configuration Guide</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="spark/dynamic_allocation.html">5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
+<li class="toctree-l2"><a class="reference internal" href="spark/aqe.html">5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
+<li class="toctree-l2"><a class="reference internal" href="spark/incremental_collection.html">5.3. Solution for Big Result Sets</a></li>
</ul>
</li>
</ul>
diff --git a/content/docs/latest/deployment/kyuubi_on_kubernetes.html b/content/docs/latest/deployment/kyuubi_on_kubernetes.html
index e220092..553d3ab 100644
--- a/content/docs/latest/deployment/kyuubi_on_kubernetes.html
+++ b/content/docs/latest/deployment/kyuubi_on_kubernetes.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Deploy Kyuubi On Kubernetes — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Deploy Kyuubi On Kubernetes — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/deployment/settings.html b/content/docs/latest/deployment/settings.html
index eeb7b8f..5957d7a 100644
--- a/content/docs/latest/deployment/settings.html
+++ b/content/docs/latest/deployment/settings.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Introduction to the Kyuubi Configurations System — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Introduction to the Kyuubi Configurations System — Kyuubi 1.5.1-incubating documentation</title>
@@ -573,7 +573,7 @@
<tbody>
<tr>
<td><code>kyuubi.engine.connection.url.use.hostname</code></td>
-<td><div style='width: 65pt;word-wrap: break-word;white-space: normal'>false</div></td>
+<td><div style='width: 65pt;word-wrap: break-word;white-space: normal'>true</div></td>
<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) When true, engine register with hostname to zookeeper. When spark run on k8s with cluster mode, set to false to ensure that server can connect to engine</div></td>
<td><div style='width: 30pt'>boolean</div></td>
<td><div style='width: 20pt'>1.3.0</div></td>
@@ -805,7 +805,7 @@
</tr>
<tr>
<td><code>kyuubi.frontend.connection.url.use.hostname</code></td>
-<td><div style='width: 65pt;word-wrap: break-word;white-space: normal'>false</div></td>
+<td><div style='width: 65pt;word-wrap: break-word;white-space: normal'>true</div></td>
<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>When true, frontend services prefer hostname, otherwise, ip address</div></td>
<td><div style='width: 30pt'>boolean</div></td>
<td><div style='width: 20pt'>1.5.0</div></td>
diff --git a/content/docs/latest/deployment/spark/aqe.html b/content/docs/latest/deployment/spark/aqe.html
index 7dc91cc..840ce54 100644
--- a/content/docs/latest/deployment/spark/aqe.html
+++ b/content/docs/latest/deployment/spark/aqe.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi — Kyuubi 1.5.0-incubating documentation</title>
+ <title>5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi — Kyuubi 1.5.1-incubating documentation</title>
@@ -41,8 +41,8 @@
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="3. Solution for Big Result Sets" href="incremental_collection.html" />
- <link rel="prev" title="1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi" href="dynamic_allocation.html" />
+ <link rel="next" title="5.3. Solution for Big Result Sets" href="incremental_collection.html" />
+ <link rel="prev" title="5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi" href="dynamic_allocation.html" />
</head>
<body class="wy-body-for-nav">
@@ -101,10 +101,10 @@
<li class="toctree-l3"><a class="reference internal" href="../engine_on_kubernetes.html">2. Deploy Kyuubi engines on Kubernetes</a></li>
<li class="toctree-l3"><a class="reference internal" href="../engine_share_level.html">3. The Share Level Of Kyuubi Engines</a></li>
<li class="toctree-l3"><a class="reference internal" href="../engine_lifecycle.html">4. The TTL Of Kyuubi Engines</a></li>
-<li class="toctree-l3 current"><a class="reference internal" href="index.html">The Spark SQL Engine Configuration Guide</a><ul class="current">
-<li class="toctree-l4"><a class="reference internal" href="dynamic_allocation.html">1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
-<li class="toctree-l4 current"><a class="current reference internal" href="#">2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
-<li class="toctree-l4"><a class="reference internal" href="incremental_collection.html">3. Solution for Big Result Sets</a></li>
+<li class="toctree-l3 current"><a class="reference internal" href="index.html">5. The Spark SQL Engine Configuration Guide</a><ul class="current">
+<li class="toctree-l4"><a class="reference internal" href="dynamic_allocation.html">5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
+<li class="toctree-l4 current"><a class="current reference internal" href="#">5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
+<li class="toctree-l4"><a class="reference internal" href="incremental_collection.html">5.3. Solution for Big Result Sets</a></li>
</ul>
</li>
</ul>
@@ -180,9 +180,9 @@
<li><a href="../index.html">Deploying Kyuubi</a> »</li>
- <li><a href="index.html">The Spark SQL Engine Configuration Guide</a> »</li>
+ <li><a href="index.html"><span class="section-number">5. </span>The Spark SQL Engine Configuration Guide</a> »</li>
- <li><span class="section-number">2. </span>How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</li>
+ <li><span class="section-number">5.2. </span>How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</li>
<li class="wy-breadcrumbs-aside">
@@ -218,9 +218,9 @@
- limitations under the License.
--><div align=center><p><img alt="../../_images/kyuubi_logo.png" src="../../_images/kyuubi_logo.png" /></p>
</div><div class="section" id="how-to-use-spark-adaptive-query-execution-aqe-in-kyuubi">
-<h1><span class="section-number">2. </span>How To Use Spark Adaptive Query Execution (AQE) in Kyuubi<a class="headerlink" href="#how-to-use-spark-adaptive-query-execution-aqe-in-kyuubi" title="Permalink to this headline">¶</a></h1>
+<h1><span class="section-number">5.2. </span>How To Use Spark Adaptive Query Execution (AQE) in Kyuubi<a class="headerlink" href="#how-to-use-spark-adaptive-query-execution-aqe-in-kyuubi" title="Permalink to this headline">¶</a></h1>
<div class="section" id="the-basics-of-aqe">
-<h2><span class="section-number">2.1. </span>The Basics of AQE<a class="headerlink" href="#the-basics-of-aqe" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.2.1. </span>The Basics of AQE<a class="headerlink" href="#the-basics-of-aqe" title="Permalink to this headline">¶</a></h2>
<p>Spark Adaptive Query Execution (AQE) is a query re-optimization that occurs during query execution.</p>
<p>In terms of technical architecture, the AQE is a framework of dynamic planning and replanning of queries based on runtime statistics,
which supports a variety of optimizations such as,</p>
@@ -233,7 +233,7 @@ which supports a variety of optimizations such as,</p>
<div class="section" id="dynamically-switch-join-strategies">
<h3>Dynamically Switch Join Strategies<a class="headerlink" href="#dynamically-switch-join-strategies" title="Permalink to this headline">¶</a></h3>
<p>Spark supports several join strategies, among which <code class="docutils literal notranslate"><span class="pre">BroadcastHash</span> <span class="pre">Join</span></code> is usually the most performant when any join side fits well in memory. And for this reason, Spark plans a <code class="docutils literal notranslate"><span class="pre">BroadcastHash</span> <span class="pre">Join</span></code> if the estimated size of a join relation is less than the <code class="docutils literal notra [...]
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.autoBroadcastJoinThreshold</span><span class="o">=</span><span class="s">10M</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.autoBroadcastJoinThreshold</span><span class="o">=</span><span class="s">10M</span><span class="w"></span>
</pre></div>
</div>
<p>Without AQE, the estimated size of join relations comes from the statistics of the original table. It can go wrong in most real-world cases. For example, the join relation is a convergent but composite operation rather than a single table scan. In this case, Spark might not be able to switch the join-strategy to <code class="docutils literal notranslate"><span class="pre">BroadcastHash</span> <span class="pre">Join</span></code>. While with AQE, we can runtime calculate the size of t [...]
@@ -272,16 +272,16 @@ which supports a variety of optimizations such as,</p>
</em>
</p><p>It can simplify the tuning of shuffle partition numbers when running Spark SQL queries. You do not need to set a proper shuffle partition number to fit your dataset.</p>
<p>To enable this feature, we need to set the below two configs to true.</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.adaptive.enabled</span><span class="o">=</span><span class="s">true</span>
-<span class="na">spark.sql.adaptive.coalescePartitions.enabled</span><span class="o">=</span><span class="s">true</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.adaptive.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
+<span class="na">spark.sql.adaptive.coalescePartitions.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
</pre></div>
</div>
<div class="section" id="other-tips-for-best-practises">
<h4>Other Tips for Best Practises<a class="headerlink" href="#other-tips-for-best-practises" title="Permalink to this headline">¶</a></h4>
<p>For further tuning our Spark jobs with this feature, we also need to be aware of these configs.</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.adaptive.advisoryPartitionSizeInBytes</span><span class="o">=</span><span class="s">128m</span>
-<span class="na">spark.sql.adaptive.coalescePartitions.minPartitionNum</span><span class="o">=</span><span class="s">1</span>
-<span class="na">spark.sql.adaptive.coalescePartitions.initialPartitionNum</span><span class="o">=</span><span class="s">200</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.adaptive.advisoryPartitionSizeInBytes</span><span class="o">=</span><span class="s">128m</span><span class="w"></span>
+<span class="na">spark.sql.adaptive.coalescePartitions.minPartitionNum</span><span class="o">=</span><span class="s">1</span><span class="w"></span>
+<span class="na">spark.sql.adaptive.coalescePartitions.initialPartitionNum</span><span class="o">=</span><span class="s">200</span><span class="w"></span>
</pre></div>
</div>
<div class="section" id="how-to-set-spark-sql-adaptive-advisorypartitionsizeinbytes">
@@ -310,16 +310,16 @@ which supports a variety of optimizations such as,</p>
<a href="https://databricks.com/blog/2020/05/29/adaptive-query-execution-speeding-up-spark-sql-at-runtime.html">[2] From Databricks Blog</a>
</em>
</p><p>To enable this feature, we need to set the below two configs to true.</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.adaptive.enabled</span><span class="o">=</span><span class="s">true</span>
-<span class="na">spark.sql.adaptive.skewJoin.enabled</span><span class="o">=</span><span class="s">true</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.adaptive.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
+<span class="na">spark.sql.adaptive.skewJoin.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
</pre></div>
</div>
<div class="section" id="id1">
<h4>Other Tips for Best Practises<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h4>
<p>For further tuning our Spark jobs with this feature, we also need to be aware of these configs.</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.adaptive.skewJoin.skewedPartitionFactor</span><span class="o">=</span><span class="s">5</span>
-<span class="na">spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</span><span class="o">=</span><span class="s">256M</span>
-<span class="na">spark.sql.adaptive.advisoryPartitionSizeInBytes</span><span class="o">=</span><span class="s">64M</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.sql.adaptive.skewJoin.skewedPartitionFactor</span><span class="o">=</span><span class="s">5</span><span class="w"></span>
+<span class="na">spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</span><span class="o">=</span><span class="s">256M</span><span class="w"></span>
+<span class="na">spark.sql.adaptive.advisoryPartitionSizeInBytes</span><span class="o">=</span><span class="s">64M</span><span class="w"></span>
</pre></div>
</div>
<div class="section" id="how-to-set-spark-sql-adaptive-skewjoin-skewedpartitionfactor-and-skewedpartitionthresholdinbytes">
@@ -350,14 +350,14 @@ skewedPartitionThresholdInBytes
<div class="section" id="disabling-the-hidden-features">
<h4>Disabling the Hidden Features<a class="headerlink" href="#disabling-the-hidden-features" title="Permalink to this headline">¶</a></h4>
<p>We can exclude some of the AQE additional rules if performance regression or bug occurs. For example,</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SET</span> <span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">adaptive</span><span class="p">.</span><span class="n">optimizer</span><span class="p">.</span><span class="n">excludedRules</span><span class="o">=</span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span cla [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SET</span><span class="w"> </span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">adaptive</span><span class="p">.</span><span class="n">optimizer</span><span class="p">.</span><span class="n">excludedRules</span><span class="o">=</span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span clas [...]
</pre></div>
</div>
</div>
</div>
</div>
<div class="section" id="best-practices-for-applying-aqe-to-kyuubi">
-<h2><span class="section-number">2.2. </span>Best Practices for Applying AQE to Kyuubi<a class="headerlink" href="#best-practices-for-applying-aqe-to-kyuubi" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.2.2. </span>Best Practices for Applying AQE to Kyuubi<a class="headerlink" href="#best-practices-for-applying-aqe-to-kyuubi" title="Permalink to this headline">¶</a></h2>
<p>Kyuubi is a long-running service to make it easier for end-users to use Spark SQL without having much of Spark’s basic knowledge. It is essential to have a basic configuration that works for most scenarios on the server-side.</p>
<div class="section" id="setting-default-configurations">
<h3>Setting Default Configurations<a class="headerlink" href="#setting-default-configurations" title="Permalink to this headline">¶</a></h3>
@@ -396,13 +396,13 @@ Since AQE requires at least one shuffle, ideally, we need to set <code class="do
</div>
</div>
<div class="section" id="spark-known-issues">
-<h2><span class="section-number">2.3. </span>Spark Known issues<a class="headerlink" href="#spark-known-issues" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.2.3. </span>Spark Known issues<a class="headerlink" href="#spark-known-issues" title="Permalink to this headline">¶</a></h2>
<p><a class="reference external" href="https://issues.apache.org/jira/browse/SPARK-33933">SPARK-33933: Broadcast timeout happened unexpectedly in AQE</a></p>
<p>For Spark versions(<3.1), we need to increase <code class="docutils literal notranslate"><span class="pre">spark.sql.broadcastTimeout(300s)</span></code> higher even the broadcast relation is tiny.</p>
<p>For other potential problems that may be found in the AQE features of Spark, you may refer to <a class="reference external" href="https://issues.apache.org/jira/browse/SPARK-33828">SPARK-33828: SQL Adaptive Query Execution QA</a>.</p>
</div>
<div class="section" id="references">
-<h2><span class="section-number">2.4. </span>References<a class="headerlink" href="#references" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.2.4. </span>References<a class="headerlink" href="#references" title="Permalink to this headline">¶</a></h2>
<ol class="simple">
<li><p><a class="reference external" href="https://spark.apache.org/docs/latest/sql-performance-tuning.html#adaptive-query-execution">Adaptive Query Execution</a></p></li>
<li><p><a class="reference external" href="https://databricks.com/blog/2020/05/29/adaptive-query-execution-speeding-up-spark-sql-at-runtime.html">Adaptive Query Execution: Speeding Up Spark SQL at Runtime</a></p></li>
@@ -419,8 +419,8 @@ Since AQE requires at least one shuffle, ideally, we need to set <code class="do
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
- <a href="incremental_collection.html" class="btn btn-neutral float-right" title="3. Solution for Big Result Sets" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
- <a href="dynamic_allocation.html" class="btn btn-neutral float-left" title="1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+ <a href="incremental_collection.html" class="btn btn-neutral float-right" title="5.3. Solution for Big Result Sets" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+ <a href="dynamic_allocation.html" class="btn btn-neutral float-left" title="5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
</div>
<hr/>
diff --git a/content/docs/latest/deployment/spark/basics.html b/content/docs/latest/deployment/spark/basics.html
index 31b9e11..d422b64 100644
--- a/content/docs/latest/deployment/spark/basics.html
+++ b/content/docs/latest/deployment/spark/basics.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title><no title> — Kyuubi 1.5.0-incubating documentation</title>
+ <title><no title> — Kyuubi 1.5.1-incubating documentation</title>
@@ -193,19 +193,19 @@
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Spark Configurations, they will override those in $SPARK_HOME/conf/spark-defaults.conf</span>
-<span class="c">## Dummy Ones</span>
-<span class="c"># spark.master local</span>
-<span class="c"># spark.submit.deployMode client</span>
-<span class="c"># spark.ui.enabled false</span>
-<span class="c"># spark.ui.port 0</span>
-<span class="c"># spark.scheduler.mode FAIR</span>
-<span class="c"># spark.serializer org.apache.spark.serializer.KryoSerializer</span>
-<span class="c"># spark.kryoserializer.buffer.max 128m</span>
-<span class="c"># spark.buffer.size 131072</span>
-<span class="c"># spark.local.dir ./local</span>
-<span class="c"># spark.network.timeout 120s</span>
-<span class="c"># spark.cleaner.periodicGC.interval 10min</span>
+ --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Spark Configurations, they will override those in $SPARK_HOME/conf/spark-defaults.conf</span><span class="w"></span>
+<span class="c">## Dummy Ones</span><span class="w"></span>
+<span class="c"># spark.master local</span><span class="w"></span>
+<span class="c"># spark.submit.deployMode client</span><span class="w"></span>
+<span class="c"># spark.ui.enabled false</span><span class="w"></span>
+<span class="c"># spark.ui.port 0</span><span class="w"></span>
+<span class="c"># spark.scheduler.mode FAIR</span><span class="w"></span>
+<span class="c"># spark.serializer org.apache.spark.serializer.KryoSerializer</span><span class="w"></span>
+<span class="c"># spark.kryoserializer.buffer.max 128m</span><span class="w"></span>
+<span class="c"># spark.buffer.size 131072</span><span class="w"></span>
+<span class="c"># spark.local.dir ./local</span><span class="w"></span>
+<span class="c"># spark.network.timeout 120s</span><span class="w"></span>
+<span class="c"># spark.cleaner.periodicGC.interval 10min</span><span class="w"></span>
</pre></div>
</div>
diff --git a/content/docs/latest/deployment/spark/driver.html b/content/docs/latest/deployment/spark/driver.html
index 271282f..4adb858 100644
--- a/content/docs/latest/deployment/spark/driver.html
+++ b/content/docs/latest/deployment/spark/driver.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title><no title> — Kyuubi 1.5.0-incubating documentation</title>
+ <title><no title> — Kyuubi 1.5.1-incubating documentation</title>
@@ -193,16 +193,16 @@
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Spark Driver / AM Sizing</span>
-<span class="c"># spark.driver.cores 4</span>
-<span class="c"># spark.driver.memory 8g</span>
-<span class="c"># spark.driver.memoryOverhead 2048</span>
-<span class="c"># spark.driver.extraJavaOptions -XX:MaxDirectMemorySize=2048m</span>
-<span class="c"># spark.driver.maxResultSize 3g</span>
-<span class="c"># spark.driver.extraJavaOptions -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005</span>
-<span class="c"># spark.yarn.am.cores 4</span>
-<span class="c"># spark.yarn.am.memory 2g</span>
-<span class="c"># spark.yarn.am.memoryOverhead 1024</span>
+ --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Spark Driver / AM Sizing</span><span class="w"></span>
+<span class="c"># spark.driver.cores 4</span><span class="w"></span>
+<span class="c"># spark.driver.memory 8g</span><span class="w"></span>
+<span class="c"># spark.driver.memoryOverhead 2048</span><span class="w"></span>
+<span class="c"># spark.driver.extraJavaOptions -XX:MaxDirectMemorySize=2048m</span><span class="w"></span>
+<span class="c"># spark.driver.maxResultSize 3g</span><span class="w"></span>
+<span class="c"># spark.driver.extraJavaOptions -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005</span><span class="w"></span>
+<span class="c"># spark.yarn.am.cores 4</span><span class="w"></span>
+<span class="c"># spark.yarn.am.memory 2g</span><span class="w"></span>
+<span class="c"># spark.yarn.am.memoryOverhead 1024</span><span class="w"></span>
</pre></div>
</div>
diff --git a/content/docs/latest/deployment/spark/dynamic_allocation.html b/content/docs/latest/deployment/spark/dynamic_allocation.html
index 54cf849..5b921b7 100644
--- a/content/docs/latest/deployment/spark/dynamic_allocation.html
+++ b/content/docs/latest/deployment/spark/dynamic_allocation.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi — Kyuubi 1.5.0-incubating documentation</title>
+ <title>5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi — Kyuubi 1.5.1-incubating documentation</title>
@@ -41,8 +41,8 @@
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi" href="aqe.html" />
- <link rel="prev" title="The Spark SQL Engine Configuration Guide" href="index.html" />
+ <link rel="next" title="5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi" href="aqe.html" />
+ <link rel="prev" title="5. The Spark SQL Engine Configuration Guide" href="index.html" />
</head>
<body class="wy-body-for-nav">
@@ -101,10 +101,10 @@
<li class="toctree-l3"><a class="reference internal" href="../engine_on_kubernetes.html">2. Deploy Kyuubi engines on Kubernetes</a></li>
<li class="toctree-l3"><a class="reference internal" href="../engine_share_level.html">3. The Share Level Of Kyuubi Engines</a></li>
<li class="toctree-l3"><a class="reference internal" href="../engine_lifecycle.html">4. The TTL Of Kyuubi Engines</a></li>
-<li class="toctree-l3 current"><a class="reference internal" href="index.html">The Spark SQL Engine Configuration Guide</a><ul class="current">
-<li class="toctree-l4 current"><a class="current reference internal" href="#">1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
-<li class="toctree-l4"><a class="reference internal" href="aqe.html">2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
-<li class="toctree-l4"><a class="reference internal" href="incremental_collection.html">3. Solution for Big Result Sets</a></li>
+<li class="toctree-l3 current"><a class="reference internal" href="index.html">5. The Spark SQL Engine Configuration Guide</a><ul class="current">
+<li class="toctree-l4 current"><a class="current reference internal" href="#">5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
+<li class="toctree-l4"><a class="reference internal" href="aqe.html">5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
+<li class="toctree-l4"><a class="reference internal" href="incremental_collection.html">5.3. Solution for Big Result Sets</a></li>
</ul>
</li>
</ul>
@@ -180,9 +180,9 @@
<li><a href="../index.html">Deploying Kyuubi</a> »</li>
- <li><a href="index.html">The Spark SQL Engine Configuration Guide</a> »</li>
+ <li><a href="index.html"><span class="section-number">5. </span>The Spark SQL Engine Configuration Guide</a> »</li>
- <li><span class="section-number">1. </span>How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</li>
+ <li><span class="section-number">5.1. </span>How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</li>
<li class="wy-breadcrumbs-aside">
@@ -218,7 +218,7 @@
- limitations under the License.
--><div align=center><p><img alt="../../_images/kyuubi_logo.png" src="../../_images/kyuubi_logo.png" /></p>
</div><div class="section" id="how-to-use-spark-dynamic-resource-allocation-dra-in-kyuubi">
-<h1><span class="section-number">1. </span>How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi<a class="headerlink" href="#how-to-use-spark-dynamic-resource-allocation-dra-in-kyuubi" title="Permalink to this headline">¶</a></h1>
+<h1><span class="section-number">5.1. </span>How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi<a class="headerlink" href="#how-to-use-spark-dynamic-resource-allocation-dra-in-kyuubi" title="Permalink to this headline">¶</a></h1>
<p>When we adopt Kyuubi in a production environment,
we always want to use the environment’s computing resources more cost-effectively and efficiently.
Cluster managers such as K8S and Yarn manage the cluster compute resources,
@@ -240,7 +240,7 @@ On the one hand, we need to rely on the resource manager’s capabilities for ef
resource isolation, and sharing.
On the other hand, we need to enable Spark’s DRA feature for the engines’ executors’ elastic scaling.</p>
<div class="section" id="the-basics-of-dynamic-resource-allocation">
-<h2><span class="section-number">1.1. </span>The Basics of Dynamic Resource Allocation<a class="headerlink" href="#the-basics-of-dynamic-resource-allocation" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.1.1. </span>The Basics of Dynamic Resource Allocation<a class="headerlink" href="#the-basics-of-dynamic-resource-allocation" title="Permalink to this headline">¶</a></h2>
<p>Spark provides a mechanism to dynamically adjust the application resources based on the workload, which means that an application may give resources back to the cluster if they are no longer used and request them again later when there is demand.
This feature is handy if multiple applications share resources on YARN, Kubernetes, and other platforms.</p>
<p>For Kyuubi engines,
@@ -254,7 +254,7 @@ and the occupied resources are given back to the cluster manager.
Then other engines or other applications run in the same queue could acquire the resources.</p>
</div>
<div class="section" id="how-to-enable-dynamic-resource-allocation">
-<h2><span class="section-number">1.2. </span>How to Enable Dynamic Resource Allocation<a class="headerlink" href="#how-to-enable-dynamic-resource-allocation" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.1.2. </span>How to Enable Dynamic Resource Allocation<a class="headerlink" href="#how-to-enable-dynamic-resource-allocation" title="Permalink to this headline">¶</a></h2>
<p>The prerequisite for enabling this feature is for downstream stages to have proper access to shuffle data, even if the executors that generated the data are recycled.</p>
<p>Spark provides two implementations for shuffle data tracking. If either is enabled, we can use the DRA feature properly.</p>
<div class="section" id="dynamic-resource-allocation-w-external-shuffle-service">
@@ -262,8 +262,8 @@ Then other engines or other applications run in the same queue could acquire the
<p>Having an external shuffle service (ESS) makes sure that all the data is stored outside of executors.
This prerequisite was needed as Spark needed to ensure that the executors’ removal does not remove shuffle data.
When deploying Kyuubi with a cluster manager that provides ESS, enable DRA for all the engines with the configurations below.</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.enabled</span><span class="o">=</span><span class="s">true</span>
-<span class="na">spark.shuffle.service.enabled</span><span class="o">=</span><span class="s">true</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
+<span class="na">spark.shuffle.service.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
</pre></div>
</div>
<p>Another thing to be sure of is that <code class="docutils literal notranslate"><span class="pre">spark.shuffle.service.port</span></code> should be configured to point to the port on which the ESS is running.</p>
@@ -273,15 +273,15 @@ When deploying Kyuubi with a cluster manager that provides ESS, enable DRA for a
<p>Implementations of the ESS feature are cluster manager dependent. Yarn, for instance, where the ESS needs to be deployed cluster-widely and is actually running in the Yarn’s <code class="docutils literal notranslate"><span class="pre">NodeManager</span></code> component. Nevertheless, if run Kyuubi’s engines on Kubernetes, the ESS is not an option yet.
Since Spark 3.0, the DRA can run without ESS. The relative feature called <code class="docutils literal notranslate"><span class="pre">Shuffle</span> <span class="pre">Tracking</span></code> was introduced by <a class="reference external" href="https://issues.apache.org/jira/browse/SPARK-27963">SPARK-27963</a>.</p>
<p>When deploying Kyuubi with a cluster manager that without ESS or the ESS is not attractive, enable DRA with <code class="docutils literal notranslate"><span class="pre">Shuffle</span> <span class="pre">Tracking</span></code> instead for all the engines with the configurations below.</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.enabled</span><span class="o">=</span><span class="s">true</span>
-<span class="na">spark.dynamicAllocation.shuffleTracking.enabled</span><span class="o">=</span><span class="s">true</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.shuffleTracking.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
</pre></div>
</div>
<p>When <code class="docutils literal notranslate"><span class="pre">Shuffle</span> <span class="pre">Tracking</span></code> is enabled, <code class="docutils literal notranslate"><span class="pre">spark.dynamicAllocation.shuffleTracking.timeout(default:</span> <span class="pre">infinity)</span></code> controls the timeout for executors that are holding shuffle data. Spark will rely on the shuffles being garbage collected to be able to release executors by default. When the garbage colle [...]
</div>
</div>
<div class="section" id="sizing-for-engines-w-dynamic-resource-allocation">
-<h2><span class="section-number">1.3. </span>Sizing for engines w/ Dynamic Resource Allocation<a class="headerlink" href="#sizing-for-engines-w-dynamic-resource-allocation" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.1.3. </span>Sizing for engines w/ Dynamic Resource Allocation<a class="headerlink" href="#sizing-for-engines-w-dynamic-resource-allocation" title="Permalink to this headline">¶</a></h2>
<p>Resources for a single executor, such as CPUs and memory, can be fixed size. So, the range [<code class="docutils literal notranslate"><span class="pre">minExecutors</span></code>, <code class="docutils literal notranslate"><span class="pre">maxExecutors</span></code>] determines how many recourses the engine can take from the cluster manager.</p>
<p>On the one hand, the <code class="docutils literal notranslate"><span class="pre">minExecutors</span></code> tells Spark to keep how many executors at least. If it is set too close to 0(default), the engine might complain about a lack of resources if the cluster manager is quite busy and for a long time.
However, the larger the <code class="docutils literal notranslate"><span class="pre">minExecutors</span></code> goes, the more resources may be wasted during the engine’s idle time.</p>
@@ -295,7 +295,7 @@ However, the larger the <code class="docutils literal notranslate"><span class="
<p>Ideally, the size relationship between them should be as <code class="docutils literal notranslate"><span class="pre">minExecutors</span></code> <= <code class="docutils literal notranslate"><span class="pre">initialExecutors</span></code> < <code class="docutils literal notranslate"><span class="pre">maxExecutors</span></code>.</p>
</div>
<div class="section" id="resource-allocation-policy">
-<h2><span class="section-number">1.4. </span>Resource Allocation Policy<a class="headerlink" href="#resource-allocation-policy" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.1.4. </span>Resource Allocation Policy<a class="headerlink" href="#resource-allocation-policy" title="Permalink to this headline">¶</a></h2>
<p>When the DRA notices that the current resources are insufficient for the current workload, it will request more executors.</p>
<div align=center><p><img alt="../../_images/dra_task_pending.png" src="../../_images/dra_task_pending.png" /></p>
</div><p>By default, the dynamic allocation will request enough executors to maximize the parallelism according to the number of tasks to process.</p>
@@ -306,39 +306,39 @@ However, the larger the <code class="docutils literal notranslate"><span class="
</div><p>After finish one task, Spark Driver will schedule a new task for the executor with available cores. When pending tasks become fewer and fewer, some executors become idle for no new coming tasks.</p>
<div align=center><p><img alt="../../_images/dra_task_fin.png" src="../../_images/dra_task_fin.png" /></p>
</div><p>If one executor reached the maximum timeout, it will be removed.</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.executorIdleTimeout</span><span class="o">=</span><span class="s">60s</span>
-<span class="na">spark.dynamicAllocation.cachedExecutorIdleTimeout</span><span class="o">=</span><span class="s">infinity</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.executorIdleTimeout</span><span class="o">=</span><span class="s">60s</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.cachedExecutorIdleTimeout</span><span class="o">=</span><span class="s">infinity</span><span class="w"></span>
</pre></div>
</div>
<div align=center><p><img alt="../../_images/dra_executor_removal.png" src="../../_images/dra_executor_removal.png" /></p>
</div><p>If the DRA finds there have been pending tasks backlogged for more than the timeouts, new executors will be requested, controlled by the following configs.</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.schedulerBacklogTimeout</span><span class="o">=</span><span class="s">1s</span>
-<span class="na">spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</span><span class="o">=</span><span class="s">1s</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.schedulerBacklogTimeout</span><span class="o">=</span><span class="s">1s</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</span><span class="o">=</span><span class="s">1s</span><span class="w"></span>
</pre></div>
</div>
</div>
<div class="section" id="best-practices-for-applying-dra-to-kyuubi">
-<h2><span class="section-number">1.5. </span>Best Practices for Applying DRA to Kyuubi<a class="headerlink" href="#best-practices-for-applying-dra-to-kyuubi" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.1.5. </span>Best Practices for Applying DRA to Kyuubi<a class="headerlink" href="#best-practices-for-applying-dra-to-kyuubi" title="Permalink to this headline">¶</a></h2>
<p>Kyuubi is a long-running service to make it easier for end-users to use Spark SQL without having much of Spark’s basic knowledge. It is essential to have a basic configuration for resource management that works for most scenarios on the server-side.</p>
<div class="section" id="setting-default-configurations">
<h3>Setting Default Configurations<a class="headerlink" href="#setting-default-configurations" title="Permalink to this headline">¶</a></h3>
<p><a class="reference external" href="settings.html#via-spark-defaults-conf">Configuring by <code class="docutils literal notranslate"><span class="pre">spark-defaults.conf</span></code></a> at the engine side is the best way to set up Kyuubi with DRA. All engines will be instantiated with DRA enabled.</p>
<p>Here is a config setting that we use in our platform when deploying Kyuubi.</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.enabled</span><span class="o">=</span><span class="s">true</span>
-<span class="c">##false if perfer shuffle tracking than ESS</span>
-<span class="na">spark.shuffle.service.enabled</span><span class="o">=</span><span class="s">true</span>
-<span class="na">spark.dynamicAllocation.initialExecutors</span><span class="o">=</span><span class="s">10</span>
-<span class="na">spark.dynamicAllocation.minExecutors</span><span class="o">=</span><span class="s">10</span>
-<span class="na">spark.dynamicAllocation.maxExecutors</span><span class="o">=</span><span class="s">500</span>
-<span class="na">spark.dynamicAllocation.executorAllocationRatio</span><span class="o">=</span><span class="s">0.5</span>
-<span class="na">spark.dynamicAllocation.executorIdleTimeout</span><span class="o">=</span><span class="s">60s</span>
-<span class="na">spark.dynamicAllocation.cachedExecutorIdleTimeout</span><span class="o">=</span><span class="s">30min</span>
-<span class="c"># true if perfer shuffle tracking than ESS</span>
-<span class="na">spark.dynamicAllocation.shuffleTracking.enabled</span><span class="o">=</span><span class="s">false</span>
-<span class="na">spark.dynamicAllocation.shuffleTracking.timeout</span><span class="o">=</span><span class="s">30min</span>
-<span class="na">spark.dynamicAllocation.schedulerBacklogTimeout</span><span class="o">=</span><span class="s">1s</span>
-<span class="na">spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</span><span class="o">=</span><span class="s">1s</span>
-<span class="na">spark.cleaner.periodicGC.interval</span><span class="o">=</span><span class="s">5min</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">spark.dynamicAllocation.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
+<span class="c">##false if perfer shuffle tracking than ESS</span><span class="w"></span>
+<span class="na">spark.shuffle.service.enabled</span><span class="o">=</span><span class="s">true</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.initialExecutors</span><span class="o">=</span><span class="s">10</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.minExecutors</span><span class="o">=</span><span class="s">10</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.maxExecutors</span><span class="o">=</span><span class="s">500</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.executorAllocationRatio</span><span class="o">=</span><span class="s">0.5</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.executorIdleTimeout</span><span class="o">=</span><span class="s">60s</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.cachedExecutorIdleTimeout</span><span class="o">=</span><span class="s">30min</span><span class="w"></span>
+<span class="c"># true if perfer shuffle tracking than ESS</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.shuffleTracking.enabled</span><span class="o">=</span><span class="s">false</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.shuffleTracking.timeout</span><span class="o">=</span><span class="s">30min</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.schedulerBacklogTimeout</span><span class="o">=</span><span class="s">1s</span><span class="w"></span>
+<span class="na">spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</span><span class="o">=</span><span class="s">1s</span><span class="w"></span>
+<span class="na">spark.cleaner.periodicGC.interval</span><span class="o">=</span><span class="s">5min</span><span class="w"></span>
</pre></div>
</div>
<p>Note that, <code class="docutils literal notranslate"><span class="pre">spark.cleaner.periodicGC.interval=5min</span></code> is useful here when <code class="docutils literal notranslate"><span class="pre">spark.dynamicAllocation.shuffleTracking.enabled</span></code> is enabled, as we can tell Spark to be more active for shuffle data GC.</p>
@@ -347,10 +347,10 @@ However, the larger the <code class="docutils literal notranslate"><span class="
<h3>Setting User Default Settings<a class="headerlink" href="#setting-user-default-settings" title="Permalink to this headline">¶</a></h3>
<p>On the server-side, the workloads for different users might be different.</p>
<p>Then we can set different defaults for them via the <a class="reference external" href="../settings.html#user-defaults">User Defaults</a> in <code class="docutils literal notranslate"><span class="pre">$KYUUBI_HOME/conf/kyuubi-defaults.conf</span></code></p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c"># For a user named kent</span>
-<span class="na">___kent___.spark.dynamicAllocation.maxExecutors</span><span class="o">=</span><span class="s">20</span>
-<span class="c"># For a user named bob</span>
-<span class="na">___bob___.spark.dynamicAllocation.maxExecutors</span><span class="o">=</span><span class="s">600</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c"># For a user named kent</span><span class="w"></span>
+<span class="na">___kent___.spark.dynamicAllocation.maxExecutors</span><span class="o">=</span><span class="s">20</span><span class="w"></span>
+<span class="c"># For a user named bob</span><span class="w"></span>
+<span class="na">___bob___.spark.dynamicAllocation.maxExecutors</span><span class="o">=</span><span class="s">600</span><span class="w"></span>
</pre></div>
</div>
<p>In this case, the user named <code class="docutils literal notranslate"><span class="pre">kent</span></code> can only use 20 executors for his engines, but <code class="docutils literal notranslate"><span class="pre">bob</span></code> can use 600 executors for better performance or handle heavy workloads.</p>
@@ -358,8 +358,8 @@ However, the larger the <code class="docutils literal notranslate"><span class="
<div class="section" id="dynamically-setting">
<h3>Dynamically Setting<a class="headerlink" href="#dynamically-setting" title="Permalink to this headline">¶</a></h3>
<p>All AQE related configurations are static of Spark core and unchangeable by <code class="docutils literal notranslate"><span class="pre">SET</span></code> syntaxes before each SQL query. For example,</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SET</span> <span class="n">spark</span><span class="p">.</span><span class="n">dynamicAllocation</span><span class="p">.</span><span class="n">maxExecutors</span><span class="o">=</span><span class="mi">33</span><span class="p">;</span>
-<span class="k">SELECT</span> <span class="o">*</span> <span class="k">FROM</span> <span class="k">default</span><span class="p">.</span><span class="n">tableA</span><span class="p">;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SET</span><span class="w"> </span><span class="n">spark</span><span class="p">.</span><span class="n">dynamicAllocation</span><span class="p">.</span><span class="n">maxExecutors</span><span class="o">=</span><span class="mi">33</span><span class="p">;</span><span class="w"></span>
+<span class="k">SELECT</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="k">FROM</span><span class="w"> </span><span class="k">default</span><span class="p">.</span><span class="n">tableA</span><span class="p">;</span><span class="w"></span>
</pre></div>
</div>
<p>For the above case, the value - 33 will not affect as Spark does not support change core configurations in runtime.</p>
@@ -367,7 +367,7 @@ However, the larger the <code class="docutils literal notranslate"><span class="
</div>
</div>
<div class="section" id="references">
-<h2><span class="section-number">1.6. </span>References<a class="headerlink" href="#references" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.1.6. </span>References<a class="headerlink" href="#references" title="Permalink to this headline">¶</a></h2>
<ol class="simple">
<li><p><a class="reference external" href="https://spark.apache.org/docs/latest/job-scheduling.html#dynamic-resource-allocation">Spark Official Online Document: Dynamic Resource Allocation</a></p></li>
<li><p><a class="reference external" href="https://spark.apache.org/docs/latest/configuration.html#dynamic-allocation">Spark Official Online Document: Dynamic Resource Allocation Configurations</a></p></li>
@@ -382,8 +382,8 @@ However, the larger the <code class="docutils literal notranslate"><span class="
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
- <a href="aqe.html" class="btn btn-neutral float-right" title="2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
- <a href="index.html" class="btn btn-neutral float-left" title="The Spark SQL Engine Configuration Guide" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+ <a href="aqe.html" class="btn btn-neutral float-right" title="5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+ <a href="index.html" class="btn btn-neutral float-left" title="5. The Spark SQL Engine Configuration Guide" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
</div>
<hr/>
diff --git a/content/docs/latest/deployment/spark/dynamicpartitionpruning.html b/content/docs/latest/deployment/spark/dynamicpartitionpruning.html
index c7b75d9..4c1bee1 100644
--- a/content/docs/latest/deployment/spark/dynamicpartitionpruning.html
+++ b/content/docs/latest/deployment/spark/dynamicpartitionpruning.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Dynamic Partition Pruning — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Dynamic Partition Pruning — Kyuubi 1.5.1-incubating documentation</title>
@@ -195,10 +195,10 @@
- limitations under the License.
--><div class="section" id="dynamic-partition-pruning">
<h1>Dynamic Partition Pruning<a class="headerlink" href="#dynamic-partition-pruning" title="Permalink to this headline">¶</a></h1>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c"># spark.sql.optimizer.dynamicPartitionPruning.enabled true</span>
-<span class="c"># spark.sql.optimizer.dynamicPartitionPruning.useStats true</span>
-<span class="c"># spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio 0.5</span>
-<span class="c"># spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly true</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c"># spark.sql.optimizer.dynamicPartitionPruning.enabled true</span><span class="w"></span>
+<span class="c"># spark.sql.optimizer.dynamicPartitionPruning.useStats true</span><span class="w"></span>
+<span class="c"># spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio 0.5</span><span class="w"></span>
+<span class="c"># spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly true</span><span class="w"></span>
</pre></div>
</div>
</div>
diff --git a/content/docs/latest/deployment/spark/ess.html b/content/docs/latest/deployment/spark/ess.html
index e336bcc..c6638d0 100644
--- a/content/docs/latest/deployment/spark/ess.html
+++ b/content/docs/latest/deployment/spark/ess.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title><no title> — Kyuubi 1.5.0-incubating documentation</title>
+ <title><no title> — Kyuubi 1.5.1-incubating documentation</title>
@@ -193,10 +193,10 @@
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## External Shuffle Service</span>
-<span class="c"># spark.shuffle.service.enabled true</span>
-<span class="c"># spark.shuffle.service.fetch.rdd.enabled true</span>
-<span class="c"># spark.shuffle.service.port 7337</span>
+ --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## External Shuffle Service</span><span class="w"></span>
+<span class="c"># spark.shuffle.service.enabled true</span><span class="w"></span>
+<span class="c"># spark.shuffle.service.fetch.rdd.enabled true</span><span class="w"></span>
+<span class="c"># spark.shuffle.service.port 7337</span><span class="w"></span>
</pre></div>
</div>
diff --git a/content/docs/latest/deployment/spark/eventqueue.html b/content/docs/latest/deployment/spark/eventqueue.html
index 32bebdf..824d9d7 100644
--- a/content/docs/latest/deployment/spark/eventqueue.html
+++ b/content/docs/latest/deployment/spark/eventqueue.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Event Queue Capacity — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Event Queue Capacity — Kyuubi 1.5.1-incubating documentation</title>
@@ -195,12 +195,12 @@
- limitations under the License.
--><div class="section" id="event-queue-capacity">
<h1>Event Queue Capacity<a class="headerlink" href="#event-queue-capacity" title="Permalink to this headline">¶</a></h1>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c"># spark.scheduler.revive.interval 1s</span>
-<span class="c"># spark.scheduler.listenerbus.eventqueue.capacity 100000</span>
-<span class="c"># spark.scheduler.listenerbus.eventqueue.executorManagement.capacity 100000</span>
-<span class="c"># spark.scheduler.listenerbus.eventqueue.appStatus.capacity 100000</span>
-<span class="c"># spark.scheduler.listenerbus.eventqueue.shared.capacity 100000</span>
-<span class="c"># spark.scheduler.listenerbus.eventqueue.eventLog.capacity 20000</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c"># spark.scheduler.revive.interval 1s</span><span class="w"></span>
+<span class="c"># spark.scheduler.listenerbus.eventqueue.capacity 100000</span><span class="w"></span>
+<span class="c"># spark.scheduler.listenerbus.eventqueue.executorManagement.capacity 100000</span><span class="w"></span>
+<span class="c"># spark.scheduler.listenerbus.eventqueue.appStatus.capacity 100000</span><span class="w"></span>
+<span class="c"># spark.scheduler.listenerbus.eventqueue.shared.capacity 100000</span><span class="w"></span>
+<span class="c"># spark.scheduler.listenerbus.eventqueue.eventLog.capacity 20000</span><span class="w"></span>
</pre></div>
</div>
</div>
diff --git a/content/docs/latest/deployment/spark/executor.html b/content/docs/latest/deployment/spark/executor.html
index bc62e1e..b7a2184 100644
--- a/content/docs/latest/deployment/spark/executor.html
+++ b/content/docs/latest/deployment/spark/executor.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title><no title> — Kyuubi 1.5.0-incubating documentation</title>
+ <title><no title> — Kyuubi 1.5.1-incubating documentation</title>
@@ -193,12 +193,12 @@
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Spark Executor Sizing</span>
-<span class="c"># spark.executor.instances 100</span>
-<span class="c"># spark.executor.cores 4</span>
-<span class="c"># spark.executor.memory 16g</span>
-<span class="c"># spark.executor.memoryOverhead 4096</span>
-<span class="c"># spark.executor.extraJavaOptions -XX:MaxDirectMemorySize=2048m</span>
+ --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Spark Executor Sizing</span><span class="w"></span>
+<span class="c"># spark.executor.instances 100</span><span class="w"></span>
+<span class="c"># spark.executor.cores 4</span><span class="w"></span>
+<span class="c"># spark.executor.memory 16g</span><span class="w"></span>
+<span class="c"># spark.executor.memoryOverhead 4096</span><span class="w"></span>
+<span class="c"># spark.executor.extraJavaOptions -XX:MaxDirectMemorySize=2048m</span><span class="w"></span>
</pre></div>
</div>
diff --git a/content/docs/latest/deployment/spark/heartbeart.html b/content/docs/latest/deployment/spark/heartbeart.html
index fffcc1f..d3bc8fc 100644
--- a/content/docs/latest/deployment/spark/heartbeart.html
+++ b/content/docs/latest/deployment/spark/heartbeart.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Executor Heartbeat — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Executor Heartbeat — Kyuubi 1.5.1-incubating documentation</title>
@@ -195,9 +195,9 @@
- limitations under the License.
--><div class="section" id="executor-heartbeat">
<h1>Executor Heartbeat<a class="headerlink" href="#executor-heartbeat" title="Permalink to this headline">¶</a></h1>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c"># spark.storage.blockManagerHeartbeatTimeoutMs 300s</span>
-<span class="c"># spark.executor.heartbeatInterval 15s</span>
-<span class="c"># spark.executor.heartbeat.maxFailures 30</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c"># spark.storage.blockManagerHeartbeatTimeoutMs 300s</span><span class="w"></span>
+<span class="c"># spark.executor.heartbeatInterval 15s</span><span class="w"></span>
+<span class="c"># spark.executor.heartbeat.maxFailures 30</span><span class="w"></span>
</pre></div>
</div>
</div>
diff --git a/content/docs/latest/deployment/spark/incremental_collection.html b/content/docs/latest/deployment/spark/incremental_collection.html
index 97b3594..a0f3a6c 100644
--- a/content/docs/latest/deployment/spark/incremental_collection.html
+++ b/content/docs/latest/deployment/spark/incremental_collection.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. Solution for Big Result Sets — Kyuubi 1.5.0-incubating documentation</title>
+ <title>5.3. Solution for Big Result Sets — Kyuubi 1.5.1-incubating documentation</title>
@@ -42,7 +42,7 @@
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<link rel="next" title="Kyuubi Security Overview" href="../../security/index.html" />
- <link rel="prev" title="2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi" href="aqe.html" />
+ <link rel="prev" title="5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi" href="aqe.html" />
</head>
<body class="wy-body-for-nav">
@@ -101,10 +101,10 @@
<li class="toctree-l3"><a class="reference internal" href="../engine_on_kubernetes.html">2. Deploy Kyuubi engines on Kubernetes</a></li>
<li class="toctree-l3"><a class="reference internal" href="../engine_share_level.html">3. The Share Level Of Kyuubi Engines</a></li>
<li class="toctree-l3"><a class="reference internal" href="../engine_lifecycle.html">4. The TTL Of Kyuubi Engines</a></li>
-<li class="toctree-l3 current"><a class="reference internal" href="index.html">The Spark SQL Engine Configuration Guide</a><ul class="current">
-<li class="toctree-l4"><a class="reference internal" href="dynamic_allocation.html">1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
-<li class="toctree-l4"><a class="reference internal" href="aqe.html">2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
-<li class="toctree-l4 current"><a class="current reference internal" href="#">3. Solution for Big Result Sets</a></li>
+<li class="toctree-l3 current"><a class="reference internal" href="index.html">5. The Spark SQL Engine Configuration Guide</a><ul class="current">
+<li class="toctree-l4"><a class="reference internal" href="dynamic_allocation.html">5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
+<li class="toctree-l4"><a class="reference internal" href="aqe.html">5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
+<li class="toctree-l4 current"><a class="current reference internal" href="#">5.3. Solution for Big Result Sets</a></li>
</ul>
</li>
</ul>
@@ -180,9 +180,9 @@
<li><a href="../index.html">Deploying Kyuubi</a> »</li>
- <li><a href="index.html">The Spark SQL Engine Configuration Guide</a> »</li>
+ <li><a href="index.html"><span class="section-number">5. </span>The Spark SQL Engine Configuration Guide</a> »</li>
- <li><span class="section-number">3. </span>Solution for Big Result Sets</li>
+ <li><span class="section-number">5.3. </span>Solution for Big Result Sets</li>
<li class="wy-breadcrumbs-aside">
@@ -218,7 +218,7 @@
- limitations under the License.
--><div align=center><p><img alt="../../_images/kyuubi_logo.png" src="../../_images/kyuubi_logo.png" /></p>
</div><div class="section" id="solution-for-big-result-sets">
-<h1><span class="section-number">3. </span>Solution for Big Result Sets<a class="headerlink" href="#solution-for-big-result-sets" title="Permalink to this headline">¶</a></h1>
+<h1><span class="section-number">5.3. </span>Solution for Big Result Sets<a class="headerlink" href="#solution-for-big-result-sets" title="Permalink to this headline">¶</a></h1>
<p>Typically, when a user submits a SELECT query to Spark SQL engine, the Driver calls <code class="docutils literal notranslate"><span class="pre">collect</span></code> to trigger calculation and
collect the entire data set of all tasks(a.k.a. partitions of an RDD), after all partitions data arrived, then the
client pulls the result set from the Driver through the Kyuubi Server in small batch.</p>
@@ -227,7 +227,7 @@ client pulls the result set from the Driver through the Kyuubi Server in small b
query has result set in several GB. But what if the result set size is dozens GB or event hundreds GB? It would be best
if you have incremental collection mode.</p>
<div class="section" id="incremental-collection">
-<h2><span class="section-number">3.1. </span>Incremental collection<a class="headerlink" href="#incremental-collection" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.3.1. </span>Incremental collection<a class="headerlink" href="#incremental-collection" title="Permalink to this headline">¶</a></h2>
<p>Since v1.4.0-incubating, Kyuubi supports incremental collection mode, it is a solution for big result sets. This feature
is disabled in default, you can turn on it by setting the configuration <code class="docutils literal notranslate"><span class="pre">kyuubi.operation.incremental.collect</span></code> to <code class="docutils literal notranslate"><span class="pre">true</span></code>.</p>
<p>The incremental collection changes the gather method from <code class="docutils literal notranslate"><span class="pre">collect</span></code> to <code class="docutils literal notranslate"><span class="pre">toLocalIterator</span></code>. <code class="docutils literal notranslate"><span class="pre">toLocalIterator</span></code> is a Spark
@@ -240,7 +240,7 @@ one partition of data in Driver memory. Therefore, it is still important to cont
avoid OOM.</p>
</div>
<div class="section" id="use-in-single-connections">
-<h2><span class="section-number">3.2. </span>Use in single connections<a class="headerlink" href="#use-in-single-connections" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.3.2. </span>Use in single connections<a class="headerlink" href="#use-in-single-connections" title="Permalink to this headline">¶</a></h2>
<p>As above explains, the incremental collection mode is not suitable for common query sense, you can enable incremental
collection mode for specific queries by using</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">beeline</span> <span class="o">-</span><span class="n">u</span> <span class="s1">'jdbc:hive2://kyuubi:10009/?spark.driver.maxResultSize=8g;spark.driver.memory=12g#kyuubi.engine.share.level=CONNECTION;kyuubi.operation.incremental.collect=true'</span> \
@@ -252,7 +252,7 @@ collection mode for specific queries by using</p>
being displayed, which may cause client side OOM.</p>
</div>
<div class="section" id="change-incremental-collection-mode-in-session">
-<h2><span class="section-number">3.3. </span>Change incremental collection mode in session<a class="headerlink" href="#change-incremental-collection-mode-in-session" title="Permalink to this headline">¶</a></h2>
+<h2><span class="section-number">5.3.3. </span>Change incremental collection mode in session<a class="headerlink" href="#change-incremental-collection-mode-in-session" title="Permalink to this headline">¶</a></h2>
<p>The configuration <code class="docutils literal notranslate"><span class="pre">kyuubi.operation.incremental.collect</span></code> can also be changed using <code class="docutils literal notranslate"><span class="pre">SET</span></code> in session.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">~</span> <span class="n">beeline</span> <span class="o">-</span><span class="n">u</span> <span class="s1">'jdbc:hive2://localhost:10009'</span>
<span class="n">Connected</span> <span class="n">to</span><span class="p">:</span> <span class="n">Apache</span> <span class="n">Kyuubi</span> <span class="p">(</span><span class="n">Incubating</span><span class="p">)</span> <span class="p">(</span><span class="n">version</span> <span class="mf">1.5.0</span><span class="o">-</span><span class="n">SNAPSHOT</span><span class="p">)</span>
@@ -321,7 +321,7 @@ normal mode, only produces 1 job (in blue square).</p>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="../../security/index.html" class="btn btn-neutral float-right" title="Kyuubi Security Overview" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
- <a href="aqe.html" class="btn btn-neutral float-left" title="2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+ <a href="aqe.html" class="btn btn-neutral float-left" title="5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
</div>
<hr/>
diff --git a/content/docs/latest/deployment/spark/index.html b/content/docs/latest/deployment/spark/index.html
index db70b63..bb807dd 100644
--- a/content/docs/latest/deployment/spark/index.html
+++ b/content/docs/latest/deployment/spark/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>The Spark SQL Engine Configuration Guide — Kyuubi 1.5.0-incubating documentation</title>
+ <title>5. The Spark SQL Engine Configuration Guide — Kyuubi 1.5.1-incubating documentation</title>
@@ -41,7 +41,7 @@
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi" href="dynamic_allocation.html" />
+ <link rel="next" title="5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi" href="dynamic_allocation.html" />
<link rel="prev" title="4. The TTL Of Kyuubi Engines" href="../engine_lifecycle.html" />
</head>
@@ -101,10 +101,10 @@
<li class="toctree-l3"><a class="reference internal" href="../engine_on_kubernetes.html">2. Deploy Kyuubi engines on Kubernetes</a></li>
<li class="toctree-l3"><a class="reference internal" href="../engine_share_level.html">3. The Share Level Of Kyuubi Engines</a></li>
<li class="toctree-l3"><a class="reference internal" href="../engine_lifecycle.html">4. The TTL Of Kyuubi Engines</a></li>
-<li class="toctree-l3 current"><a class="current reference internal" href="#">The Spark SQL Engine Configuration Guide</a><ul>
-<li class="toctree-l4"><a class="reference internal" href="dynamic_allocation.html">1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
-<li class="toctree-l4"><a class="reference internal" href="aqe.html">2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
-<li class="toctree-l4"><a class="reference internal" href="incremental_collection.html">3. Solution for Big Result Sets</a></li>
+<li class="toctree-l3 current"><a class="current reference internal" href="#">5. The Spark SQL Engine Configuration Guide</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="dynamic_allocation.html">5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a></li>
+<li class="toctree-l4"><a class="reference internal" href="aqe.html">5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a></li>
+<li class="toctree-l4"><a class="reference internal" href="incremental_collection.html">5.3. Solution for Big Result Sets</a></li>
</ul>
</li>
</ul>
@@ -180,7 +180,7 @@
<li><a href="../index.html">Deploying Kyuubi</a> »</li>
- <li>The Spark SQL Engine Configuration Guide</li>
+ <li><span class="section-number">5. </span>The Spark SQL Engine Configuration Guide</li>
<li class="wy-breadcrumbs-aside">
@@ -201,7 +201,7 @@
<img alt="../../_images/kyuubi_logo.png" class="align-center" src="../../_images/kyuubi_logo.png" />
<div class="section" id="the-spark-sql-engine-configuration-guide">
-<h1>The Spark SQL Engine Configuration Guide<a class="headerlink" href="#the-spark-sql-engine-configuration-guide" title="Permalink to this headline">¶</a></h1>
+<h1><span class="section-number">5. </span>The Spark SQL Engine Configuration Guide<a class="headerlink" href="#the-spark-sql-engine-configuration-guide" title="Permalink to this headline">¶</a></h1>
<p>Kyuubi aims to bring Spark to end-users who need not qualify with Spark or something else related to the big data area.
End-users can write SQL queries through JDBC against Kyuubi and nothing more.
The Kyuubi server-side or the corresponding engines could do most of the optimization.
@@ -209,26 +209,26 @@ On the other hand, we don’t wholly restrict end-users to special handling of s
Even if you don’t use Kyuubi, as a simple Spark user, I’m sure you’ll find the next articles instructive.</p>
<div class="toctree-wrapper compound">
<ul>
-<li class="toctree-l1"><a class="reference internal" href="dynamic_allocation.html">1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#the-basics-of-dynamic-resource-allocation">1.1. The Basics of Dynamic Resource Allocation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#how-to-enable-dynamic-resource-allocation">1.2. How to Enable Dynamic Resource Allocation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#sizing-for-engines-w-dynamic-resource-allocation">1.3. Sizing for engines w/ Dynamic Resource Allocation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#resource-allocation-policy">1.4. Resource Allocation Policy</a></li>
-<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#best-practices-for-applying-dra-to-kyuubi">1.5. Best Practices for Applying DRA to Kyuubi</a></li>
-<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#references">1.6. References</a></li>
+<li class="toctree-l1"><a class="reference internal" href="dynamic_allocation.html">5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#the-basics-of-dynamic-resource-allocation">5.1.1. The Basics of Dynamic Resource Allocation</a></li>
+<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#how-to-enable-dynamic-resource-allocation">5.1.2. How to Enable Dynamic Resource Allocation</a></li>
+<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#sizing-for-engines-w-dynamic-resource-allocation">5.1.3. Sizing for engines w/ Dynamic Resource Allocation</a></li>
+<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#resource-allocation-policy">5.1.4. Resource Allocation Policy</a></li>
+<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#best-practices-for-applying-dra-to-kyuubi">5.1.5. Best Practices for Applying DRA to Kyuubi</a></li>
+<li class="toctree-l2"><a class="reference internal" href="dynamic_allocation.html#references">5.1.6. References</a></li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="aqe.html">2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="aqe.html#the-basics-of-aqe">2.1. The Basics of AQE</a></li>
-<li class="toctree-l2"><a class="reference internal" href="aqe.html#best-practices-for-applying-aqe-to-kyuubi">2.2. Best Practices for Applying AQE to Kyuubi</a></li>
-<li class="toctree-l2"><a class="reference internal" href="aqe.html#spark-known-issues">2.3. Spark Known issues</a></li>
-<li class="toctree-l2"><a class="reference internal" href="aqe.html#references">2.4. References</a></li>
+<li class="toctree-l1"><a class="reference internal" href="aqe.html">5.2. How To Use Spark Adaptive Query Execution (AQE) in Kyuubi</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="aqe.html#the-basics-of-aqe">5.2.1. The Basics of AQE</a></li>
+<li class="toctree-l2"><a class="reference internal" href="aqe.html#best-practices-for-applying-aqe-to-kyuubi">5.2.2. Best Practices for Applying AQE to Kyuubi</a></li>
+<li class="toctree-l2"><a class="reference internal" href="aqe.html#spark-known-issues">5.2.3. Spark Known issues</a></li>
+<li class="toctree-l2"><a class="reference internal" href="aqe.html#references">5.2.4. References</a></li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="incremental_collection.html">3. Solution for Big Result Sets</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="incremental_collection.html#incremental-collection">3.1. Incremental collection</a></li>
-<li class="toctree-l2"><a class="reference internal" href="incremental_collection.html#use-in-single-connections">3.2. Use in single connections</a></li>
-<li class="toctree-l2"><a class="reference internal" href="incremental_collection.html#change-incremental-collection-mode-in-session">3.3. Change incremental collection mode in session</a></li>
+<li class="toctree-l1"><a class="reference internal" href="incremental_collection.html">5.3. Solution for Big Result Sets</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="incremental_collection.html#incremental-collection">5.3.1. Incremental collection</a></li>
+<li class="toctree-l2"><a class="reference internal" href="incremental_collection.html#use-in-single-connections">5.3.2. Use in single connections</a></li>
+<li class="toctree-l2"><a class="reference internal" href="incremental_collection.html#change-incremental-collection-mode-in-session">5.3.3. Change incremental collection mode in session</a></li>
</ul>
</li>
</ul>
@@ -241,7 +241,7 @@ Even if you don’t use Kyuubi, as a simple Spark user, I’m sure you’ll find
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
- <a href="dynamic_allocation.html" class="btn btn-neutral float-right" title="1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+ <a href="dynamic_allocation.html" class="btn btn-neutral float-right" title="5.1. How To Use Spark Dynamic Resource Allocation (DRA) in Kyuubi" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
<a href="../engine_lifecycle.html" class="btn btn-neutral float-left" title="4. The TTL Of Kyuubi Engines" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
</div>
diff --git a/content/docs/latest/deployment/spark/locality.html b/content/docs/latest/deployment/spark/locality.html
index 0ba3624..356b971 100644
--- a/content/docs/latest/deployment/spark/locality.html
+++ b/content/docs/latest/deployment/spark/locality.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title><no title> — Kyuubi 1.5.0-incubating documentation</title>
+ <title><no title> — Kyuubi 1.5.1-incubating documentation</title>
@@ -193,11 +193,11 @@
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Data Locality for Task Schedule</span>
-<span class="c"># spark.locality.wait 0s</span>
-<span class="c"># spark.locality.wait.process 0s</span>
-<span class="c"># spark.locality.wait.node 0s</span>
-<span class="c"># spark.locality.wait.rack 0s</span>
+ --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Data Locality for Task Schedule</span><span class="w"></span>
+<span class="c"># spark.locality.wait 0s</span><span class="w"></span>
+<span class="c"># spark.locality.wait.process 0s</span><span class="w"></span>
+<span class="c"># spark.locality.wait.node 0s</span><span class="w"></span>
+<span class="c"># spark.locality.wait.rack 0s</span><span class="w"></span>
</pre></div>
</div>
diff --git a/content/docs/latest/deployment/spark/monitering.html b/content/docs/latest/deployment/spark/monitering.html
index f7414df..d7a7318 100644
--- a/content/docs/latest/deployment/spark/monitering.html
+++ b/content/docs/latest/deployment/spark/monitering.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title><no title> — Kyuubi 1.5.0-incubating documentation</title>
+ <title><no title> — Kyuubi 1.5.1-incubating documentation</title>
@@ -193,13 +193,13 @@
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Event Logging for History Server</span>
-<span class="c"># spark.eventLog.enabled true</span>
-<span class="c"># spark.eventLog.dir hdfs://hadoop-dfs/history</span>
-<span class="c"># spark.eventLog.compress true</span>
-<span class="c"># spark.eventLog.longForm.enabled true</span>
-<span class="c"># spark.eventLog.rolling.enabled true</span>
-<span class="c"># spark.yarn.historyServer.address http://historyserver:18080</span>
+ --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Event Logging for History Server</span><span class="w"></span>
+<span class="c"># spark.eventLog.enabled true</span><span class="w"></span>
+<span class="c"># spark.eventLog.dir hdfs://hadoop-dfs/history</span><span class="w"></span>
+<span class="c"># spark.eventLog.compress true</span><span class="w"></span>
+<span class="c"># spark.eventLog.longForm.enabled true</span><span class="w"></span>
+<span class="c"># spark.eventLog.rolling.enabled true</span><span class="w"></span>
+<span class="c"># spark.yarn.historyServer.address http://historyserver:18080</span><span class="w"></span>
</pre></div>
</div>
diff --git a/content/docs/latest/deployment/spark/shuffle.html b/content/docs/latest/deployment/spark/shuffle.html
index 20d878a..e444fa7 100644
--- a/content/docs/latest/deployment/spark/shuffle.html
+++ b/content/docs/latest/deployment/spark/shuffle.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title><no title> — Kyuubi 1.5.0-incubating documentation</title>
+ <title><no title> — Kyuubi 1.5.1-incubating documentation</title>
@@ -193,33 +193,33 @@
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Shuffle Behavior</span>
-<span class="c"># spark.shuffle.compress true</span>
-<span class="c"># spark.shuffle.detectCorrupt true</span>
-<span class="c"># spark.shuffle.detectCorrupt.useExtraMemory true</span>
-<span class="c"># spark.shuffle.file.buffer 64k</span>
-<span class="c"># spark.shuffle.unsafe.file.output.buffer 64k</span>
-<span class="c"># spark.shuffle.spill.diskWriteBufferSize 8k</span>
-<span class="c"># spark.shuffle.spill.compress true</span>
-<span class="c"># spark.shuffle.mapOutput.dispatcher.numThreads 12</span>
-<span class="c"># spark.shuffle.mapOutput.parallelAggregationThreshold 5000</span>
-<span class="c"># spark.shuffle.readHostLocalDisk true</span>
-<span class="c"># spark.shuffle.io.maxRetries 10</span>
-<span class="c"># spark.shuffle.io.retryWait 6s</span>
-<span class="c"># spark.shuffle.io.preferDirectBufs false</span>
-<span class="c"># spark.shuffle.io.serverThreads 8</span>
-<span class="c"># spark.shuffle.io.clientThreads 8</span>
-<span class="c"># spark.shuffle.io.connectionTimeout 240s</span>
-<span class="c"># spark.shuffle.registration.timeout 6000</span>
-<span class="c"># spark.shuffle.registration.maxAttempts 10</span>
-<span class="c"># spark.shuffle.sync false</span>
-<span class="c"># spark.shuffle.useOldFetchProtocol true</span>
-<span class="c"># spark.shuffle.unsafe.fastMergeEnabled true</span>
-<span class="c"># spark.shuffle.minNumPartitionsToHighlyCompress 100</span>
-<span class="c"># spark.network.maxRemoteBlockSizeFetchToMem 128m</span>
-<span class="c"># spark.reducer.maxSizeInFlight 48m</span>
-<span class="c"># spark.reducer.maxReqsInFlight 256</span>
-<span class="c"># spark.reducer.maxBlocksInFlightPerAddress 256</span>
+ --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Shuffle Behavior</span><span class="w"></span>
+<span class="c"># spark.shuffle.compress true</span><span class="w"></span>
+<span class="c"># spark.shuffle.detectCorrupt true</span><span class="w"></span>
+<span class="c"># spark.shuffle.detectCorrupt.useExtraMemory true</span><span class="w"></span>
+<span class="c"># spark.shuffle.file.buffer 64k</span><span class="w"></span>
+<span class="c"># spark.shuffle.unsafe.file.output.buffer 64k</span><span class="w"></span>
+<span class="c"># spark.shuffle.spill.diskWriteBufferSize 8k</span><span class="w"></span>
+<span class="c"># spark.shuffle.spill.compress true</span><span class="w"></span>
+<span class="c"># spark.shuffle.mapOutput.dispatcher.numThreads 12</span><span class="w"></span>
+<span class="c"># spark.shuffle.mapOutput.parallelAggregationThreshold 5000</span><span class="w"></span>
+<span class="c"># spark.shuffle.readHostLocalDisk true</span><span class="w"></span>
+<span class="c"># spark.shuffle.io.maxRetries 10</span><span class="w"></span>
+<span class="c"># spark.shuffle.io.retryWait 6s</span><span class="w"></span>
+<span class="c"># spark.shuffle.io.preferDirectBufs false</span><span class="w"></span>
+<span class="c"># spark.shuffle.io.serverThreads 8</span><span class="w"></span>
+<span class="c"># spark.shuffle.io.clientThreads 8</span><span class="w"></span>
+<span class="c"># spark.shuffle.io.connectionTimeout 240s</span><span class="w"></span>
+<span class="c"># spark.shuffle.registration.timeout 6000</span><span class="w"></span>
+<span class="c"># spark.shuffle.registration.maxAttempts 10</span><span class="w"></span>
+<span class="c"># spark.shuffle.sync false</span><span class="w"></span>
+<span class="c"># spark.shuffle.useOldFetchProtocol true</span><span class="w"></span>
+<span class="c"># spark.shuffle.unsafe.fastMergeEnabled true</span><span class="w"></span>
+<span class="c"># spark.shuffle.minNumPartitionsToHighlyCompress 100</span><span class="w"></span>
+<span class="c"># spark.network.maxRemoteBlockSizeFetchToMem 128m</span><span class="w"></span>
+<span class="c"># spark.reducer.maxSizeInFlight 48m</span><span class="w"></span>
+<span class="c"># spark.reducer.maxReqsInFlight 256</span><span class="w"></span>
+<span class="c"># spark.reducer.maxBlocksInFlightPerAddress 256</span><span class="w"></span>
</pre></div>
</div>
diff --git a/content/docs/latest/deployment/spark/speculation.html b/content/docs/latest/deployment/spark/speculation.html
index 995fe95..feee6d3 100644
--- a/content/docs/latest/deployment/spark/speculation.html
+++ b/content/docs/latest/deployment/spark/speculation.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title><no title> — Kyuubi 1.5.0-incubating documentation</title>
+ <title><no title> — Kyuubi 1.5.1-incubating documentation</title>
@@ -193,12 +193,12 @@
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Speculation</span>
-<span class="c"># spark.speculation true</span>
-<span class="c"># spark.speculation.interval 1s</span>
-<span class="c"># spark.speculation.multiplier 1.5</span>
-<span class="c"># spark.speculation.quantile 0.9</span>
-<span class="c"># spark.speculation.task.duration.threshold 10min</span>
+ --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## Speculation</span><span class="w"></span>
+<span class="c"># spark.speculation true</span><span class="w"></span>
+<span class="c"># spark.speculation.interval 1s</span><span class="w"></span>
+<span class="c"># spark.speculation.multiplier 1.5</span><span class="w"></span>
+<span class="c"># spark.speculation.quantile 0.9</span><span class="w"></span>
+<span class="c"># spark.speculation.task.duration.threshold 10min</span><span class="w"></span>
</pre></div>
</div>
diff --git a/content/docs/latest/deployment/spark/sql.html b/content/docs/latest/deployment/spark/sql.html
index 0f4e45a..232b3c1 100644
--- a/content/docs/latest/deployment/spark/sql.html
+++ b/content/docs/latest/deployment/spark/sql.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title><no title> — Kyuubi 1.5.0-incubating documentation</title>
+ <title><no title> — Kyuubi 1.5.1-incubating documentation</title>
@@ -193,17 +193,17 @@
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## SQL</span>
-<span class="c">## General SQL Settings</span>
-<span class="c"># spark.sql.shuffle.partitions 8192</span>
-<span class="c"># spark.sql.optimizer.inSetConversionThreshold 2</span>
-<span class="c"># spark.sql.autoBroadcastJoinThreshold 64m</span>
-<span class="c"># spark.sql.broadcastTimeout 600s</span>
-<span class="c"># spark.sql.join.preferSortMergeJoin true</span>
-<span class="c"># spark.sql.hive.metastorePartitionPruning true</span>
-<span class="c"># spark.sql.parquet.filterPushdown true</span>
-<span class="c"># spark.sql.parquet.recordLevelFilter.enabled true</span>
-<span class="c"># spark.sql.statistics.fallBackToHdfs true</span>
+ --><div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="c">## SQL</span><span class="w"></span>
+<span class="c">## General SQL Settings</span><span class="w"></span>
+<span class="c"># spark.sql.shuffle.partitions 8192</span><span class="w"></span>
+<span class="c"># spark.sql.optimizer.inSetConversionThreshold 2</span><span class="w"></span>
+<span class="c"># spark.sql.autoBroadcastJoinThreshold 64m</span><span class="w"></span>
+<span class="c"># spark.sql.broadcastTimeout 600s</span><span class="w"></span>
+<span class="c"># spark.sql.join.preferSortMergeJoin true</span><span class="w"></span>
+<span class="c"># spark.sql.hive.metastorePartitionPruning true</span><span class="w"></span>
+<span class="c"># spark.sql.parquet.filterPushdown true</span><span class="w"></span>
+<span class="c"># spark.sql.parquet.recordLevelFilter.enabled true</span><span class="w"></span>
+<span class="c"># spark.sql.statistics.fallBackToHdfs true</span><span class="w"></span>
</pre></div>
</div>
diff --git a/content/docs/latest/develop_tools/build_document.html b/content/docs/latest/develop_tools/build_document.html
index 26b5c25..6a7d804 100644
--- a/content/docs/latest/develop_tools/build_document.html
+++ b/content/docs/latest/develop_tools/build_document.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. Building Kyuubi Documentation — Kyuubi 1.5.0-incubating documentation</title>
+ <title>3. Building Kyuubi Documentation — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/develop_tools/building.html b/content/docs/latest/develop_tools/building.html
index 0c6ba99..3b2fe14 100644
--- a/content/docs/latest/develop_tools/building.html
+++ b/content/docs/latest/develop_tools/building.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Building Kyuubi — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Building Kyuubi — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/develop_tools/debugging.html b/content/docs/latest/develop_tools/debugging.html
index 3848712..2df54d9 100644
--- a/content/docs/latest/develop_tools/debugging.html
+++ b/content/docs/latest/develop_tools/debugging.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>5. Debugging Kyuubi — Kyuubi 1.5.0-incubating documentation</title>
+ <title>5. Debugging Kyuubi — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/develop_tools/developer.html b/content/docs/latest/develop_tools/developer.html
index 094cc6d..8626557 100644
--- a/content/docs/latest/develop_tools/developer.html
+++ b/content/docs/latest/develop_tools/developer.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>6. Developer Tools — Kyuubi 1.5.0-incubating documentation</title>
+ <title>6. Developer Tools — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/develop_tools/distribution.html b/content/docs/latest/develop_tools/distribution.html
index 6510d84..38aa5ae 100644
--- a/content/docs/latest/develop_tools/distribution.html
+++ b/content/docs/latest/develop_tools/distribution.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Building a Runnable Distribution — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Building a Runnable Distribution — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/develop_tools/index.html b/content/docs/latest/develop_tools/index.html
index d8bbdb9..1609567 100644
--- a/content/docs/latest/develop_tools/index.html
+++ b/content/docs/latest/develop_tools/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Develop Tools — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Develop Tools — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/develop_tools/testing.html b/content/docs/latest/develop_tools/testing.html
index c8f1477..e712de6 100644
--- a/content/docs/latest/develop_tools/testing.html
+++ b/content/docs/latest/develop_tools/testing.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>4. Running Tests — Kyuubi 1.5.0-incubating documentation</title>
+ <title>4. Running Tests — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/genindex.html b/content/docs/latest/genindex.html
index c591abc..e20020f 100644
--- a/content/docs/latest/genindex.html
+++ b/content/docs/latest/genindex.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Index — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Index — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/index.html b/content/docs/latest/index.html
index 13adfce..99aa6fd 100644
--- a/content/docs/latest/index.html
+++ b/content/docs/latest/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Welcome to Kyuubi’s documentation! — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Welcome to Kyuubi’s documentation! — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/integrations/delta_lake.html b/content/docs/latest/integrations/delta_lake.html
index 23840af..055c90e 100644
--- a/content/docs/latest/integrations/delta_lake.html
+++ b/content/docs/latest/integrations/delta_lake.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Kyuubi On Delta Lake — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Kyuubi On Delta Lake — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/integrations/delta_lake_with_azure_blob.html b/content/docs/latest/integrations/delta_lake_with_azure_blob.html
index 1f985c3..ff22b66 100644
--- a/content/docs/latest/integrations/delta_lake_with_azure_blob.html
+++ b/content/docs/latest/integrations/delta_lake_with_azure_blob.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. Kyuubi On Delta Lake With Microsoft Azure Blob Storage — Kyuubi 1.5.0-incubating documentation</title>
+ <title>3. Kyuubi On Delta Lake With Microsoft Azure Blob Storage — Kyuubi 1.5.1-incubating documentation</title>
@@ -481,25 +481,25 @@ So, you can check the registration details in zookeeper path ’/kyuubi_USER/ano
<h2><span class="section-number">3.4. </span>Dealing Delta Lake Data By Using Kyuubi Examples<a class="headerlink" href="#dealing-delta-lake-data-by-using-kyuubi-examples" title="Permalink to this headline">¶</a></h2>
<p>Operate delta-lake data through SQL:<br />1.Create Table</p>
<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="c1">-- Create or replace table with path</span>
-<span class="k">CREATE</span> <span class="k">OR</span> <span class="k">REPLACE</span> <span class="k">TABLE</span> <span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">blob</span><span class="p">.</span><span class="n">core</span><span class="p">.</span><span cl [...]
- <span class="nb">date</span> <span class="nb">DATE</span><span class="p">,</span>
- <span class="n">eventId</span> <span class="n">STRING</span><span class="p">,</span>
- <span class="n">eventType</span> <span class="n">STRING</span><span class="p">,</span>
- <span class="k">data</span> <span class="n">STRING</span><span class="p">)</span>
-<span class="k">USING</span> <span class="n">DELTA</span>
-<span class="n">PARTITIONED</span> <span class="k">BY</span> <span class="p">(</span><span class="nb">date</span><span class="p">);</span>
+<span class="k">CREATE</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="k">REPLACE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">bl [...]
+<span class="w"> </span><span class="nb">date</span><span class="w"> </span><span class="nb">DATE</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">eventId</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">eventType</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="k">data</span><span class="w"> </span><span class="n">STRING</span><span class="p">)</span><span class="w"></span>
+<span class="k">USING</span><span class="w"> </span><span class="n">DELTA</span><span class="w"></span>
+<span class="n">PARTITIONED</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="p">(</span><span class="nb">date</span><span class="p">);</span><span class="w"></span>
</pre></div>
</div>
<p>2.Insert Data<br />Append Mode:</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">blob</span><span class="p">.</span><span class="n">core</span><span clas [...]
- <span class="nb">date</span><span class="p">,</span>
- <span class="n">eventId</span><span class="p">,</span>
- <span class="n">eventType</span><span class="p">,</span>
- <span class="k">data</span><span class="p">)</span>
-<span class="k">VALUES</span>
- <span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'001'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'Hello World!'</span><span class="p">),</span>
- <span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'002'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'Hello World!'</span><span class="p">),</span>
- <span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'003'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'Hello World!'</span><span class="p">);</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">INSERT</span><span class="w"> </span><span class="k">INTO</span><span class="w"> </span><span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">blob</span><span class="p" [...]
+<span class="w"> </span><span class="nb">date</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">eventId</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">eventType</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="k">data</span><span class="p">)</span><span class="w"></span>
+<span class="k">VALUES</span><span class="w"> </span>
+<span class="w"> </span><span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'001'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'Hello World!'</span><span class="p">),</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'002'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'Hello World!'</span><span class="p">),</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'003'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'Hello World!'</span><span class="p">);</span><span class="w"></span>
</pre></div>
</div>
<p>Result:</p>
@@ -513,14 +513,14 @@ So, you can check the registration details in zookeeper path ’/kyuubi_USER/ano
</pre></div>
</div>
<p>Overwrite Mode:</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">INSERT</span> <span class="n">OVERWRITE</span> <span class="k">TABLE</span> <span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">blob</span><span class="p">.</span><sp [...]
- <span class="nb">date</span><span class="p">,</span>
- <span class="n">eventId</span><span class="p">,</span>
- <span class="n">eventType</span><span class="p">,</span>
- <span class="k">data</span><span class="p">)</span>
-<span class="k">VALUES</span>
-<span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'001'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'hello kyuubi'</span><span class="p">),</span>
-<span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'002'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'hello kyuubi'</span><span class="p">);</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">INSERT</span><span class="w"> </span><span class="n">OVERWRITE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span clas [...]
+<span class="w"> </span><span class="nb">date</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">eventId</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">eventType</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="k">data</span><span class="p">)</span><span class="w"></span>
+<span class="k">VALUES</span><span class="w"> </span>
+<span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'001'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'hello kyuubi'</span><span class="p">),</span><span class="w"></span>
+<span class="p">(</span><span class="n">now</span><span class="p">(),</span><span class="s1">'002'</span><span class="p">,</span><span class="s1">'test'</span><span class="p">,</span><span class="s1">'hello kyuubi'</span><span class="p">);</span><span class="w"></span>
</pre></div>
</div>
<p>Result:</p>
@@ -533,7 +533,7 @@ So, you can check the registration details in zookeeper path ’/kyuubi_USER/ano
</pre></div>
</div>
<p>Delete Table Data:</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">delete</span> <span class="k">from</span> <span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">blob</span><span class="p">.</span><span class="n">core</span><span clas [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">delete</span><span class="w"> </span><span class="k">from</span><span class="w"> </span><span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">blob</span><span class="p" [...]
</pre></div>
</div>
<p>Result:</p>
@@ -545,9 +545,9 @@ So, you can check the registration details in zookeeper path ’/kyuubi_USER/ano
</pre></div>
</div>
<p>Update table data:</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">UPDATE</span> <span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">blob</span><span class="p">.</span><span class="n">core</span><span class="p">.</span><span class="n [...]
-<span class="k">SET</span> <span class="k">data</span> <span class="o">=</span> <span class="s1">'This is a test for update data.'</span>
-<span class="k">WHERE</span> <span class="n">eventId</span> <span class="o">=</span> <span class="mi">001</span><span class="p">;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">UPDATE</span><span class="w"> </span><span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">blob</span><span class="p">.</span><span class="n">core</span><span class="p" [...]
+<span class="k">SET</span><span class="w"> </span><span class="k">data</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'This is a test for update data.'</span><span class="w"></span>
+<span class="k">WHERE</span><span class="w"> </span><span class="n">eventId</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">001</span><span class="p">;</span><span class="w"></span>
</pre></div>
</div>
<p>Result:</p>
@@ -559,7 +559,7 @@ So, you can check the registration details in zookeeper path ’/kyuubi_USER/ano
</pre></div>
</div>
<p>Select table data:</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SELECT</span> <span class="o">*</span> <span class="k">FROM</span> <span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</span><span class="nb">blob</span><span class="p">.</span><span class= [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SELECT</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="k">FROM</span><span class="w"> </span><span class="n">delta</span><span class="p">.</span><span class="o">`</span><span class="n">wasbs</span><span class="p">:</span><span class="o">//</span><span class="mi">1000</span><span class="o">@</span><span class="n">azure_account</span><span class="p">.</ [...]
</pre></div>
</div>
<p>Result:</p>
diff --git a/content/docs/latest/integrations/index.html b/content/docs/latest/integrations/index.html
index 8274fae..72e885c 100644
--- a/content/docs/latest/integrations/index.html
+++ b/content/docs/latest/integrations/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Integrations — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Integrations — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/integrations/kudu.html b/content/docs/latest/integrations/kudu.html
index 02a0045..86106c1 100644
--- a/content/docs/latest/integrations/kudu.html
+++ b/content/docs/latest/integrations/kudu.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Kyuubi On Apache Kudu — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Kyuubi On Apache Kudu — Kyuubi 1.5.1-incubating documentation</title>
@@ -258,124 +258,124 @@
</div>
<div class="section" id="register-kudu-table-as-spark-temporary-view">
<h3>Register Kudu table as Spark Temporary view<a class="headerlink" href="#register-kudu-table-as-spark-temporary-view" title="Permalink to this headline">¶</a></h3>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">CREATE</span> <span class="k">TEMPORARY</span> <span class="k">VIEW</span> <span class="n">kudutest</span>
-<span class="k">USING</span> <span class="n">kudu</span>
-<span class="k">options</span> <span class="p">(</span>
- <span class="n">kudu</span><span class="p">.</span><span class="n">master</span> <span class="ss">"ip1:port1,ip2:port2,..."</span><span class="p">,</span>
- <span class="n">kudu</span><span class="p">.</span><span class="k">table</span> <span class="ss">"kudu::test.testtbl"</span><span class="p">)</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">TEMPORARY</span><span class="w"> </span><span class="k">VIEW</span><span class="w"> </span><span class="n">kudutest</span><span class="w"></span>
+<span class="k">USING</span><span class="w"> </span><span class="n">kudu</span><span class="w"></span>
+<span class="k">options</span><span class="w"> </span><span class="p">(</span><span class="w"> </span>
+<span class="w"> </span><span class="n">kudu</span><span class="p">.</span><span class="n">master</span><span class="w"> </span><span class="ss">"ip1:port1,ip2:port2,..."</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">kudu</span><span class="p">.</span><span class="k">table</span><span class="w"> </span><span class="ss">"kudu::test.testtbl"</span><span class="p">)</span><span class="w"></span>
</pre></div>
</div>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span> <span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span class="mi">10009</span><s [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Running</span> <span class="n">query</span> <span class="s1">'show tables'</span> <span class="k">with</span> [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span> <span class="n">INFO</span> <span class="n">KyuubiSparkUtil$</span><span class="p">:</span> <span class="n">Application</span> <span class="n">application_1560304876299_3805060</span> <span class="n">has</span> <span class="n">been</span> [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Executing</span> <span class="n">query</span> <span class="k">in</span> <span class="n">incremental</span> <span class [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Executing</span> <span class="n">query</span> <span class="k">in</span> <span class="n">incremental</span> <span class [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span> <span class="n">INFO</span> <span class="n">DAGScheduler</span><span class="p">:</span> <span class="n">Asked</span> <span class="k">to</span> <span class="n">cancel</span> <span class="n">job</span> <span class="k">group</span> <span cla [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span><span class="w"> </span><span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span cl [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Running</span><span class= [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">KyuubiSparkUtil$</span><span class="p">:</span><span class="w"> </span><span class="n">Application</span><span class="w"> </s [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Executing</span><span clas [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Executing</span><span clas [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">28</span><span class="p">:</span><span class="mi">03</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">DAGScheduler</span><span class="p">:</span><span class="w"> </span><span class="n">Asked</span><span class="w"> </span><span [...]
<span class="o">+</span><span class="c1">-----------+-----------------------------+--------------+--+</span>
-<span class="o">|</span> <span class="k">database</span> <span class="o">|</span> <span class="n">tableName</span> <span class="o">|</span> <span class="n">isTemporary</span> <span class="o">|</span>
+<span class="o">|</span><span class="w"> </span><span class="k">database</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">tableName</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">isTemporary</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
<span class="o">+</span><span class="c1">-----------+-----------------------------+--------------+--+</span>
-<span class="o">|</span> <span class="n">kyuubi</span> <span class="o">|</span> <span class="n">hive_tbl</span> <span class="o">|</span> <span class="k">false</span> <span class="o">|</span>
-<span class="o">|</span> <span class="o">|</span> <span class="n">kudutest</span> <span class="o">|</span> <span class="k">true</span> <span class="o">|</span>
+<span class="o">|</span><span class="w"> </span><span class="n">kyuubi</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">hive_tbl</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="k">false</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
+<span class="o">|</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">kudutest</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="k">true</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
<span class="o">+</span><span class="c1">-----------+-----------------------------+--------------+--+</span>
-<span class="mi">2</span> <span class="k">rows</span> <span class="n">selected</span> <span class="p">(</span><span class="mi">0</span><span class="p">.</span><span class="mi">29</span> <span class="n">seconds</span><span class="p">)</span>
+<span class="mi">2</span><span class="w"> </span><span class="k">rows</span><span class="w"> </span><span class="n">selected</span><span class="w"> </span><span class="p">(</span><span class="mi">0</span><span class="p">.</span><span class="mi">29</span><span class="w"> </span><span class="n">seconds</span><span class="p">)</span><span class="w"></span>
</pre></div>
</div>
</div>
<div class="section" id="query-kudu-table">
<h3>Query Kudu Table<a class="headerlink" href="#query-kudu-table" title="Permalink to this headline">¶</a></h3>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span> <span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span class="mi">10009</span><s [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Running</span> <span class="n">query</span> <span class="s1">'select * from kudutest'</span> <span class="k">w [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span> <span class="n">INFO</span> <span class="n">KyuubiSparkUtil$</span><span class="p">:</span> <span class="n">Application</span> <span class="n">application_1560304876299_3805060</span> <span class="n">has</span> <span class="n">been</span> [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Executing</span> <span class="n">query</span> <span class="k">in</span> <span class="n">incremental</span> <span class [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Executing</span> <span class="n">query</span> <span class="k">in</span> <span class="n">incremental</span> <span class [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span> <span class="n">INFO</span> <span class="n">DAGScheduler</span><span class="p">:</span> <span class="n">Asked</span> <span class="k">to</span> <span class="n">cancel</span> <span class="n">job</span> <span class="k">group</span> <span cla [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span><span class="w"> </span><span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span cl [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Running</span><span class= [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">KyuubiSparkUtil$</span><span class="p">:</span><span class="w"> </span><span class="n">Application</span><span class="w"> </s [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Executing</span><span clas [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Executing</span><span clas [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">25</span><span class="p">:</span><span class="mi">17</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">DAGScheduler</span><span class="p">:</span><span class="w"> </span><span class="n">Asked</span><span class="w"> </span><span [...]
<span class="o">+</span><span class="c1">---------+---------------+----------------+--+</span>
-<span class="o">|</span> <span class="n">userid</span> <span class="o">|</span> <span class="n">sharesetting</span> <span class="o">|</span> <span class="n">notifysetting</span> <span class="o">|</span>
+<span class="o">|</span><span class="w"> </span><span class="n">userid</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">sharesetting</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">notifysetting</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
<span class="o">+</span><span class="c1">---------+---------------+----------------+--+</span>
-<span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">5</span> <span class="o">|</span> <span class="mi">5</span> <span class="o">|</span> <span class="mi">5</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span>
+<span class="o">|</span><span class="w"> </span><span class="mi">1</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">1</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">1</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
+<span class="o">|</span><span class="w"> </span><span class="mi">5</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">5</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">5</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
+<span class="o">|</span><span class="w"> </span><span class="mi">2</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">2</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">2</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
+<span class="o">|</span><span class="w"> </span><span class="mi">3</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">3</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">3</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
+<span class="o">|</span><span class="w"> </span><span class="mi">4</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">4</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">4</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
<span class="o">+</span><span class="c1">---------+---------------+----------------+--+</span>
-<span class="mi">5</span> <span class="k">rows</span> <span class="n">selected</span> <span class="p">(</span><span class="mi">1</span><span class="p">.</span><span class="mi">083</span> <span class="n">seconds</span><span class="p">)</span>
+<span class="mi">5</span><span class="w"> </span><span class="k">rows</span><span class="w"> </span><span class="n">selected</span><span class="w"> </span><span class="p">(</span><span class="mi">1</span><span class="p">.</span><span class="mi">083</span><span class="w"> </span><span class="n">seconds</span><span class="p">)</span><span class="w"></span>
</pre></div>
</div>
</div>
<div class="section" id="join-kudu-table-with-hive-table">
<h3>Join Kudu table with Hive table<a class="headerlink" href="#join-kudu-table-with-hive-table" title="Permalink to this headline">¶</a></h3>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span> <span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span class="mi">10009</span><s [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">31</span><span class="p">:</span><span class="mi">01</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Running</span> <span class="n">query</span> <span class="s1">'select t1.*, t2.* from hive_tbl t1 join kudutest t2 [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">31</span><span class="p">:</span><span class="mi">01</span> <span class="n">INFO</span> <span class="n">KyuubiSparkUtil$</span><span class="p">:</span> <span class="n">Application</span> <span class="n">application_1560304876299_3805060</span> <span class="n">has</span> <span class="n">been</span> [...]
-<span class="o"><</span><span class="n">omitted</span> <span class="n">lines</span><span class="p">....</span> <span class="o">></span>
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">31</span><span class="p">:</span><span class="mi">01</span> <span class="n">INFO</span> <span class="n">DAGScheduler</span><span class="p">:</span> <span class="n">Asked</span> <span class="k">to</span> <span class="n">cancel</span> <span class="n">job</span> <span class="k">group</span> <span cla [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span><span class="w"> </span><span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span cl [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">31</span><span class="p">:</span><span class="mi">01</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Running</span><span class= [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">31</span><span class="p">:</span><span class="mi">01</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">KyuubiSparkUtil$</span><span class="p">:</span><span class="w"> </span><span class="n">Application</span><span class="w"> </s [...]
+<span class="o"><</span><span class="n">omitted</span><span class="w"> </span><span class="n">lines</span><span class="p">....</span><span class="w"> </span><span class="o">></span><span class="w"></span>
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">31</span><span class="p">:</span><span class="mi">01</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">DAGScheduler</span><span class="p">:</span><span class="w"> </span><span class="n">Asked</span><span class="w"> </span><span [...]
<span class="o">+</span><span class="c1">---------+---------------+----------------+---------+---------------+----------------+--+</span>
-<span class="o">|</span> <span class="n">userid</span> <span class="o">|</span> <span class="n">sharesetting</span> <span class="o">|</span> <span class="n">notifysetting</span> <span class="o">|</span> <span class="n">userid</span> <span class="o">|</span> <span class="n">sharesetting</span> <span class="o">|</span> <span class="n">notifysetting</span> <span class="o">|</span>
+<span class="o">|</span><span class="w"> </span><span class="n">userid</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">sharesetting</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">notifysetting</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">userid</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">sharese [...]
<span class="o">+</span><span class="c1">---------+---------------+----------------+---------+---------------+----------------+--+</span>
-<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
+<span class="o">|</span><span class="w"> </span><span class="mi">2</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">2</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">2</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">1</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">1< [...]
+<span class="o">|</span><span class="w"> </span><span class="mi">3</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">3</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">3</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">2</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">2< [...]
+<span class="o">|</span><span class="w"> </span><span class="mi">4</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">4</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">4</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">3</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="mi">3< [...]
<span class="o">+</span><span class="c1">---------+---------------+----------------+---------+---------------+----------------+--+</span>
-<span class="mi">3</span> <span class="k">rows</span> <span class="n">selected</span> <span class="p">(</span><span class="mi">1</span><span class="p">.</span><span class="mi">63</span> <span class="n">seconds</span><span class="p">)</span>
+<span class="mi">3</span><span class="w"> </span><span class="k">rows</span><span class="w"> </span><span class="n">selected</span><span class="w"> </span><span class="p">(</span><span class="mi">1</span><span class="p">.</span><span class="mi">63</span><span class="w"> </span><span class="n">seconds</span><span class="p">)</span><span class="w"></span>
</pre></div>
</div>
</div>
<div class="section" id="insert-to-kudu-table">
<h3>Insert to Kudu table<a class="headerlink" href="#insert-to-kudu-table" title="Permalink to this headline">¶</a></h3>
<p>You should notice that only <code class="docutils literal notranslate"><span class="pre">INSERT</span> <span class="pre">INTO</span></code> is supported by Kudu, <code class="docutils literal notranslate"><span class="pre">OVERWRITE</span></code> data is not supported</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span> <span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span class="mi">10009</span><s [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">35</span><span class="p">:</span><span class="mi">29</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Running</span> <span class="n">query</span> <span class="s1">'insert overwrite table kudutest select * from hive_ [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">35</span><span class="p">:</span><span class="mi">29</span> <span class="n">INFO</span> <span class="n">KyuubiSparkUtil$</span><span class="p">:</span> <span class="n">Application</span> <span class="n">application_1560304876299_3805060</span> <span class="n">has</span> <span class="n">been</span> [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">35</span><span class="p">:</span><span class="mi">30</span> <span class="n">ERROR</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span>
-<span class="n">Error</span> <span class="n">executing</span> <span class="n">query</span> <span class="k">as</span> <span class="n">bdms_hzyaoqin</span><span class="p">,</span>
-<span class="k">insert</span> <span class="n">overwrite</span> <span class="k">table</span> <span class="n">kudutest</span> <span class="k">select</span> <span class="o">*</span> <span class="k">from</span> <span class="n">hive_tbl</span>
-<span class="k">Current</span> <span class="k">operation</span> <span class="k">state</span> <span class="n">RUNNING</span><span class="p">,</span>
-<span class="n">java</span><span class="p">.</span><span class="n">lang</span><span class="p">.</span><span class="n">UnsupportedOperationException</span><span class="p">:</span> <span class="n">overwrite</span> <span class="k">is</span> <span class="k">not</span> <span class="n">yet</span> <span class="n">supported</span>
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kudu</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="n">kudu</span><span class="p">.</span><span class="n">KuduRelation</span><span class="p">.</span><span class="k">insert</span><span class="p">(</span><span class="n">DefaultSource</span><span class="p">.</span><span class="n">scala</span><span cl [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">datasources</span><span class="p">.</span><span class="n">InsertIntoDataSourceCommand</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">Insert [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">command</span><span class="p">.</span><span class="n">ExecutedCommandExec</span><span class="p">.</span><span class="n">sideEffectResult$lzycompute</span><span class="p">(</span><span clas [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">command</span><span class="p">.</span><span class="n">ExecutedCommandExec</span><span class="p">.</span><span class="n">sideEffectResult</span><span class="p">(</span><span class="n">comma [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">command</span><span class="p">.</span><span class="n">ExecutedCommandExec</span><span class="p">.</span><span class="n">executeCollect</span><span class="p">(</span><span class="n">command [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset$$anonfun$6</span><span class="p">.</span><span class="n">apply</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p">:</span><span class="mi">190</span><span clas [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset$$anonfun$6</span><span class="p">.</span><span class="n">apply</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p">:</span><span class="mi">190</span><span clas [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset$$anonfun$52</span><span class="p">.</span><span class="n">apply</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p">:</span><span class="mi">3259</span><span cl [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">SQLExecution$</span><span class="p">.</span><span class="n">withNewExecutionId</span><span class="p">(</span><span class="n">SQLExecution</span><span class="p">.</span><span class="n">scal [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset</span><span class="p">.</span><span class="n">withAction</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p">:</span><span class="mi">3258</span><span class="p" [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset</span><span class="p">.</span><span class="o"><</span><span class="n">init</span><span class="o">></span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p">:</ [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset$</span><span class="p">.</span><span class="n">ofRows</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p">:</span><span class="mi">75</span><span class="p">)</span>
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">SparkSQLUtils$</span><span class="p">.</span><span class="n">toDataFrame</span><span class="p">(</span><span class="n">SparkSQLUtils</span><span class="p">.</span><span class="n">scala</span><span class="p">:</span><span class="mi">39</span><sp [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kyuubi</span><span class="p">.</span><span class="k">operation</span><span class="p">.</span><span class="k">statement</span><span class="p">.</span><span class="n">ExecuteStatementInClientMode</span><span class="p">.</span><span class="k">execute</span><span class="p">(</span><span class="n">ExecuteStatementInClientMode</span><span class="p">. [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kyuubi</span><span class="p">.</span><span class="k">operation</span><span class="p">.</span><span class="k">statement</span><span class="p">.</span><span class="n">ExecuteStatementOperation$$anon$1$$anon$2</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">ExecuteStatementOperation</span><span class [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kyuubi</span><span class="p">.</span><span class="k">operation</span><span class="p">.</span><span class="k">statement</span><span class="p">.</span><span class="n">ExecuteStatementOperation$$anon$1$$anon$2</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">ExecuteStatementOperation</span><span class [...]
- <span class="k">at</span> <span class="n">java</span><span class="p">.</span><span class="k">security</span><span class="p">.</span><span class="n">AccessController</span><span class="p">.</span><span class="n">doPrivileged</span><span class="p">(</span><span class="n">Native</span> <span class="k">Method</span><span class="p">)</span>
- <span class="k">at</span> <span class="n">javax</span><span class="p">.</span><span class="k">security</span><span class="p">.</span><span class="n">auth</span><span class="p">.</span><span class="n">Subject</span><span class="p">.</span><span class="n">doAs</span><span class="p">(</span><span class="n">Subject</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">422</span><span class="p">)</span>
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="k">security</span><span class="p">.</span><span class="n">UserGroupInformation</span><span class="p">.</span><span class="n">doAs</span><span class="p">(</span><span class="n">UserGroupInformation</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">1 [...]
- <span class="k">at</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kyuubi</span><span class="p">.</span><span class="k">operation</span><span class="p">.</span><span class="k">statement</span><span class="p">.</span><span class="n">ExecuteStatementOperation$$anon$1</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">ExecuteStatementOperation</span><span class="p">.</ [...]
- <span class="k">at</span> <span class="n">java</span><span class="p">.</span><span class="n">util</span><span class="p">.</span><span class="n">concurrent</span><span class="p">.</span><span class="n">Executors$RunnableAdapter</span><span class="p">.</span><span class="k">call</span><span class="p">(</span><span class="n">Executors</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">511</span><span class="p">)</span>
- <span class="k">at</span> <span class="n">java</span><span class="p">.</span><span class="n">util</span><span class="p">.</span><span class="n">concurrent</span><span class="p">.</span><span class="n">FutureTask</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">FutureTask</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">266</span><span class="p">)</span>
- <span class="k">at</span> <span class="n">java</span><span class="p">.</span><span class="n">util</span><span class="p">.</span><span class="n">concurrent</span><span class="p">.</span><span class="n">ThreadPoolExecutor</span><span class="p">.</span><span class="n">runWorker</span><span class="p">(</span><span class="n">ThreadPoolExecutor</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">1142</span><span class="p">)</span>
- <span class="k">at</span> <span class="n">java</span><span class="p">.</span><span class="n">util</span><span class="p">.</span><span class="n">concurrent</span><span class="p">.</span><span class="n">ThreadPoolExecutor$Worker</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">ThreadPoolExecutor</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">617</span><span class="p">)</span>
- <span class="k">at</span> <span class="n">java</span><span class="p">.</span><span class="n">lang</span><span class="p">.</span><span class="n">Thread</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">Thread</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">745</span><span class="p">)</span>
-
-
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">35</span><span class="p">:</span><span class="mi">30</span> <span class="n">INFO</span> <span class="n">DAGScheduler</span><span class="p">:</span> <span class="n">Asked</span> <span class="k">to</span> <span class="n">cancel</span> <span class="n">job</span> <span class="k">group</span> <span cla [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span><span class="w"> </span><span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span cl [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">35</span><span class="p">:</span><span class="mi">29</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Running</span><span class= [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">35</span><span class="p">:</span><span class="mi">29</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">KyuubiSparkUtil$</span><span class="p">:</span><span class="w"> </span><span class="n">Application</span><span class="w"> </s [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">35</span><span class="p">:</span><span class="mi">30</span><span class="w"> </span><span class="n">ERROR</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"></span>
+<span class="n">Error</span><span class="w"> </span><span class="n">executing</span><span class="w"> </span><span class="n">query</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="n">bdms_hzyaoqin</span><span class="p">,</span><span class="w"></span>
+<span class="k">insert</span><span class="w"> </span><span class="n">overwrite</span><span class="w"> </span><span class="k">table</span><span class="w"> </span><span class="n">kudutest</span><span class="w"> </span><span class="k">select</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="k">from</span><span class="w"> </span><span class="n">hive_tbl</span><span class="w"></span>
+<span class="k">Current</span><span class="w"> </span><span class="k">operation</span><span class="w"> </span><span class="k">state</span><span class="w"> </span><span class="n">RUNNING</span><span class="p">,</span><span class="w"></span>
+<span class="n">java</span><span class="p">.</span><span class="n">lang</span><span class="p">.</span><span class="n">UnsupportedOperationException</span><span class="p">:</span><span class="w"> </span><span class="n">overwrite</span><span class="w"> </span><span class="k">is</span><span class="w"> </span><span class="k">not</span><span class="w"> </span><span class="n">yet</span><span class="w"> </span><span class="n">supported</span><span class="w"></span>
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kudu</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="n">kudu</span><span class="p">.</span><span class="n">KuduRelation</span><span class="p">.</span><span class="k">insert</span><span class="p">(</span><span class="n">DefaultSource</span><span class="p [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">datasources</span><span class="p">.</span><span class="n">InsertIntoDataSourceCommand</span><span class="p">.</span><span class="n">run</span> [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">command</span><span class="p">.</span><span class="n">ExecutedCommandExec</span><span class="p">.</span><span class="n">sideEffectResult$lzyco [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">command</span><span class="p">.</span><span class="n">ExecutedCommandExec</span><span class="p">.</span><span class="n">sideEffectResult</span [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">command</span><span class="p">.</span><span class="n">ExecutedCommandExec</span><span class="p">.</span><span class="n">executeCollect</span>< [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset$$anonfun$6</span><span class="p">.</span><span class="n">apply</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p" [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset$$anonfun$6</span><span class="p">.</span><span class="n">apply</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p" [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset$$anonfun$52</span><span class="p">.</span><span class="n">apply</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">execution</span><span class="p">.</span><span class="n">SQLExecution$</span><span class="p">.</span><span class="n">withNewExecutionId</span><span class="p">(</span><span class="n">SQLExecution</spa [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset</span><span class="p">.</span><span class="n">withAction</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p">:</sp [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset</span><span class="p">.</span><span class="o"><</span><span class="n">init</span><span class="o">></span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span>< [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">Dataset$</span><span class="p">.</span><span class="n">ofRows</span><span class="p">(</span><span class="n">Dataset</span><span class="p">.</span><span class="n">scala</span><span class="p">:</span> [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">SparkSQLUtils$</span><span class="p">.</span><span class="n">toDataFrame</span><span class="p">(</span><span class="n">SparkSQLUtils</span><span class="p">.</span><span class="n">scala</span><span c [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kyuubi</span><span class="p">.</span><span class="k">operation</span><span class="p">.</span><span class="k">statement</span><span class="p">.</span><span class="n">ExecuteStatementInClientMode</span><span class="p">.</span><span class="k">execute</span><span class="p">(</span><span class="n">Execut [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kyuubi</span><span class="p">.</span><span class="k">operation</span><span class="p">.</span><span class="k">statement</span><span class="p">.</span><span class="n">ExecuteStatementOperation$$anon$1$$anon$2</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class=" [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kyuubi</span><span class="p">.</span><span class="k">operation</span><span class="p">.</span><span class="k">statement</span><span class="p">.</span><span class="n">ExecuteStatementOperation$$anon$1$$anon$2</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class=" [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">java</span><span class="p">.</span><span class="k">security</span><span class="p">.</span><span class="n">AccessController</span><span class="p">.</span><span class="n">doPrivileged</span><span class="p">(</span><span class="n">Native</span><span class="w"> </span><span class="k">Method</span><span class="p">)</span><span class="w"></span>
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">javax</span><span class="p">.</span><span class="k">security</span><span class="p">.</span><span class="n">auth</span><span class="p">.</span><span class="n">Subject</span><span class="p">.</span><span class="n">doAs</span><span class="p">(</span><span class="n">Subject</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">422</span><span class="p">)</span> [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="k">security</span><span class="p">.</span><span class="n">UserGroupInformation</span><span class="p">.</span><span class="n">doAs</span><span class="p">(</span><span class="n">UserGroupInformation</span><span class="p">.</span><span class="n">java</s [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">kyuubi</span><span class="p">.</span><span class="k">operation</span><span class="p">.</span><span class="k">statement</span><span class="p">.</span><span class="n">ExecuteStatementOperation$$anon$1</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">Execu [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">java</span><span class="p">.</span><span class="n">util</span><span class="p">.</span><span class="n">concurrent</span><span class="p">.</span><span class="n">Executors$RunnableAdapter</span><span class="p">.</span><span class="k">call</span><span class="p">(</span><span class="n">Executors</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">511</span><sp [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">java</span><span class="p">.</span><span class="n">util</span><span class="p">.</span><span class="n">concurrent</span><span class="p">.</span><span class="n">FutureTask</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">FutureTask</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">266</span><span class="p">)< [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">java</span><span class="p">.</span><span class="n">util</span><span class="p">.</span><span class="n">concurrent</span><span class="p">.</span><span class="n">ThreadPoolExecutor</span><span class="p">.</span><span class="n">runWorker</span><span class="p">(</span><span class="n">ThreadPoolExecutor</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">1142</ [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">java</span><span class="p">.</span><span class="n">util</span><span class="p">.</span><span class="n">concurrent</span><span class="p">.</span><span class="n">ThreadPoolExecutor$Worker</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">ThreadPoolExecutor</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">617</ [...]
+<span class="w"> </span><span class="k">at</span><span class="w"> </span><span class="n">java</span><span class="p">.</span><span class="n">lang</span><span class="p">.</span><span class="n">Thread</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="n">Thread</span><span class="p">.</span><span class="n">java</span><span class="p">:</span><span class="mi">745</span><span class="p">)</span><span class="w"></span>
+
+
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">35</span><span class="p">:</span><span class="mi">30</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">DAGScheduler</span><span class="p">:</span><span class="w"> </span><span class="n">Asked</span><span class="w"> </span><span [...]
</pre></div>
</div>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span> <span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span class="mi">10009</span><s [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">26</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Running</span> <span class="n">query</span> <span class="s1">'insert into table kudutest select * from hive_tbl&# [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">26</span> <span class="n">INFO</span> <span class="n">KyuubiSparkUtil$</span><span class="p">:</span> <span class="n">Application</span> <span class="n">application_1560304876299_3805060</span> <span class="n">has</span> <span class="n">been</span> [...]
-<span class="o"><</span><span class="n">omitted</span> <span class="n">lines</span> <span class="p">...</span><span class="o">></span>
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span> <span class="n">INFO</span> <span class="n">DAGScheduler</span><span class="p">:</span> <span class="n">ResultStage</span> <span class="mi">36</span> <span class="p">(</span><span class="n">foreachPartition</span> <span class="k">at</span [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span> <span class="n">INFO</span> <span class="n">DAGScheduler</span><span class="p">:</span> <span class="n">Job</span> <span class="mi">36</span> <span class="n">finished</span><span class="p">:</span> <span class="n">foreachPartition</span> [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span> <span class="n">INFO</span> <span class="n">KuduContext</span><span class="p">:</span> <span class="n">completed</span> <span class="n">upsert</span> <span class="n">ops</span><span class="p">:</span> <span class="n">duration</span> <span [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Executing</span> <span class="n">query</span> <span class="k">in</span> <span class="n">incremental</span> <span class [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span> <span class="n">INFO</span> <span class="n">ExecuteStatementInClientMode</span><span class="p">:</span> <span class="n">Executing</span> <span class="n">query</span> <span class="k">in</span> <span class="n">incremental</span> <span class [...]
-<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span> <span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span> <span class="n">INFO</span> <span class="n">DAGScheduler</span><span class="p">:</span> <span class="n">Asked</span> <span class="k">to</span> <span class="n">cancel</span> <span class="n">job</span> <span class="k">group</span> <span cla [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span><span class="p">:</span><span class="w"> </span><span class="n">jdbc</span><span class="p">:</span><span class="n">hive2</span><span class="p">:</span><span class="o">//</span><span class="n">spark5</span><span class="p">.</span><span class="n">jd</span><span class="p">.</span><span class="mi">163</span><span class="p">.</span><span class="n">org</span><span class="p">:</span><span cl [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">26</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Running</span><span class= [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">26</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">KyuubiSparkUtil$</span><span class="p">:</span><span class="w"> </span><span class="n">Application</span><span class="w"> </s [...]
+<span class="o"><</span><span class="n">omitted</span><span class="w"> </span><span class="n">lines</span><span class="w"> </span><span class="p">...</span><span class="o">></span><span class="w"></span>
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">DAGScheduler</span><span class="p">:</span><span class="w"> </span><span class="n">ResultStage</span><span class="w"> </span> [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">DAGScheduler</span><span class="p">:</span><span class="w"> </span><span class="n">Job</span><span class="w"> </span><span cl [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">KuduContext</span><span class="p">:</span><span class="w"> </span><span class="n">completed</span><span class="w"> </span><sp [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Executing</span><span clas [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">ExecuteStatementInClientMode</span><span class="p">:</span><span class="w"> </span><span class="n">Executing</span><span clas [...]
+<span class="mi">19</span><span class="o">/</span><span class="mi">07</span><span class="o">/</span><span class="mi">09</span><span class="w"> </span><span class="mi">15</span><span class="p">:</span><span class="mi">36</span><span class="p">:</span><span class="mi">27</span><span class="w"> </span><span class="n">INFO</span><span class="w"> </span><span class="n">DAGScheduler</span><span class="p">:</span><span class="w"> </span><span class="n">Asked</span><span class="w"> </span><span [...]
<span class="o">+</span><span class="c1">---------+--+</span>
-<span class="o">|</span> <span class="k">Result</span> <span class="o">|</span>
+<span class="o">|</span><span class="w"> </span><span class="k">Result</span><span class="w"> </span><span class="o">|</span><span class="w"></span>
<span class="o">+</span><span class="c1">---------+--+</span>
<span class="o">+</span><span class="c1">---------+--+</span>
-<span class="k">No</span> <span class="k">rows</span> <span class="n">selected</span> <span class="p">(</span><span class="mi">0</span><span class="p">.</span><span class="mi">611</span> <span class="n">seconds</span><span class="p">)</span>
+<span class="k">No</span><span class="w"> </span><span class="k">rows</span><span class="w"> </span><span class="n">selected</span><span class="w"> </span><span class="p">(</span><span class="mi">0</span><span class="p">.</span><span class="mi">611</span><span class="w"> </span><span class="n">seconds</span><span class="p">)</span><span class="w"></span>
</pre></div>
</div>
</div>
diff --git a/content/docs/latest/monitor/events.html b/content/docs/latest/monitor/events.html
index 19665ad..63d0c52 100644
--- a/content/docs/latest/monitor/events.html
+++ b/content/docs/latest/monitor/events.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Monitoring Kyuubi - Events System — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Monitoring Kyuubi - Events System — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/monitor/index.html b/content/docs/latest/monitor/index.html
index 9611524..117033f 100644
--- a/content/docs/latest/monitor/index.html
+++ b/content/docs/latest/monitor/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Monitoring — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Monitoring — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/monitor/logging.html b/content/docs/latest/monitor/logging.html
index aa7926c..7832381 100644
--- a/content/docs/latest/monitor/logging.html
+++ b/content/docs/latest/monitor/logging.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Monitoring Kyuubi - Logging System — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Monitoring Kyuubi - Logging System — Kyuubi 1.5.1-incubating documentation</title>
@@ -300,13 +300,13 @@ $ <span class="nv">KYUUBI_LOG_DIR</span><span class="o">=</span>/Users/kentyao/t
<h4>Custom Log4j Settings<a class="headerlink" href="#custom-log4j-settings" title="Permalink to this headline">¶</a></h4>
<p>Taking control of <code class="docutils literal notranslate"><span class="pre">$KYUUBI_HOME/conf/log4j.properties</span></code> will also give us the ability of customizing server logging as we want.</p>
<p>For example, we can disable the console appender and enable the file appender like,</p>
-<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">log4j.rootCategory</span><span class="o">=</span><span class="s">INFO, FA</span>
-<span class="na">log4j.appender.FA</span><span class="o">=</span><span class="s">org.apache.log4j.FileAppender</span>
-<span class="na">log4j.appender.FA.append</span><span class="o">=</span><span class="s">false</span>
-<span class="na">log4j.appender.FA.file</span><span class="o">=</span><span class="s">log/dummy.log</span>
-<span class="na">log4j.appender.FA.layout</span><span class="o">=</span><span class="s">org.apache.log4j.PatternLayout</span>
-<span class="na">log4j.appender.FA.layout.ConversionPattern</span><span class="o">=</span><span class="s">%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n</span>
-<span class="na">log4j.appender.FA.Threshold</span><span class="o">=</span><span class="s">DEBUG</span>
+<div class="highlight-properties notranslate"><div class="highlight"><pre><span></span><span class="na">log4j.rootCategory</span><span class="o">=</span><span class="s">INFO, FA</span><span class="w"></span>
+<span class="na">log4j.appender.FA</span><span class="o">=</span><span class="s">org.apache.log4j.FileAppender</span><span class="w"></span>
+<span class="na">log4j.appender.FA.append</span><span class="o">=</span><span class="s">false</span><span class="w"></span>
+<span class="na">log4j.appender.FA.file</span><span class="o">=</span><span class="s">log/dummy.log</span><span class="w"></span>
+<span class="na">log4j.appender.FA.layout</span><span class="o">=</span><span class="s">org.apache.log4j.PatternLayout</span><span class="w"></span>
+<span class="na">log4j.appender.FA.layout.ConversionPattern</span><span class="o">=</span><span class="s">%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n</span><span class="w"></span>
+<span class="na">log4j.appender.FA.Threshold</span><span class="o">=</span><span class="s">DEBUG</span><span class="w"></span>
</pre></div>
</div>
<p>Then everything goes to <code class="docutils literal notranslate"><span class="pre">log/dummy.log</span></code>.</p>
diff --git a/content/docs/latest/monitor/metrics.html b/content/docs/latest/monitor/metrics.html
index fb97488..ec8ec26 100644
--- a/content/docs/latest/monitor/metrics.html
+++ b/content/docs/latest/monitor/metrics.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Monitoring Kyuubi - Server Metrics — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Monitoring Kyuubi - Server Metrics — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/monitor/trouble_shooting.html b/content/docs/latest/monitor/trouble_shooting.html
index 53aa226..eb7efff 100644
--- a/content/docs/latest/monitor/trouble_shooting.html
+++ b/content/docs/latest/monitor/trouble_shooting.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. Trouble Shooting — Kyuubi 1.5.0-incubating documentation</title>
+ <title>3. Trouble Shooting — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/overview/architecture.html b/content/docs/latest/overview/architecture.html
index 28f9331..e4923e4 100644
--- a/content/docs/latest/overview/architecture.html
+++ b/content/docs/latest/overview/architecture.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Kyuubi Architecture — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Kyuubi Architecture — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/overview/index.html b/content/docs/latest/overview/index.html
index 2c2b7d0..8357552 100644
--- a/content/docs/latest/overview/index.html
+++ b/content/docs/latest/overview/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Overview — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Overview — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/overview/kyuubi_vs_hive.html b/content/docs/latest/overview/kyuubi_vs_hive.html
index 4332dad..b82e1a7 100644
--- a/content/docs/latest/overview/kyuubi_vs_hive.html
+++ b/content/docs/latest/overview/kyuubi_vs_hive.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Kyuubi v.s. HiveServer2 — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Kyuubi v.s. HiveServer2 — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/overview/kyuubi_vs_thriftserver.html b/content/docs/latest/overview/kyuubi_vs_thriftserver.html
index d6387bc..c93afbd 100644
--- a/content/docs/latest/overview/kyuubi_vs_thriftserver.html
+++ b/content/docs/latest/overview/kyuubi_vs_thriftserver.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Kyuubi v.s. Spark Thrift JDBC/ODBC Server (STS) — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Kyuubi v.s. Spark Thrift JDBC/ODBC Server (STS) — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/quick_start/index.html b/content/docs/latest/quick_start/index.html
index 693d569..ce937af 100644
--- a/content/docs/latest/quick_start/index.html
+++ b/content/docs/latest/quick_start/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Quick Start — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Quick Start — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/quick_start/quick_start.html b/content/docs/latest/quick_start/quick_start.html
index ee1fd2b..81a81a9 100644
--- a/content/docs/latest/quick_start/quick_start.html
+++ b/content/docs/latest/quick_start/quick_start.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Getting Started with Apache Kyuubi — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Getting Started with Apache Kyuubi — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/quick_start/quick_start_with_beeline.html b/content/docs/latest/quick_start/quick_start_with_beeline.html
index b27a6bb..8defd3f 100644
--- a/content/docs/latest/quick_start/quick_start_with_beeline.html
+++ b/content/docs/latest/quick_start/quick_start_with_beeline.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>4. Getting Started With Hive Beeline — Kyuubi 1.5.0-incubating documentation</title>
+ <title>4. Getting Started With Hive Beeline — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/quick_start/quick_start_with_datagrip.html b/content/docs/latest/quick_start/quick_start_with_datagrip.html
index 2933291..c7098cb 100644
--- a/content/docs/latest/quick_start/quick_start_with_datagrip.html
+++ b/content/docs/latest/quick_start/quick_start_with_datagrip.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>6. Getting Started With Kyuubi and DataGrip — Kyuubi 1.5.0-incubating documentation</title>
+ <title>6. Getting Started With Kyuubi and DataGrip — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/quick_start/quick_start_with_dbeaver.html b/content/docs/latest/quick_start/quick_start_with_dbeaver.html
index 2a598a1..c0584a5 100644
--- a/content/docs/latest/quick_start/quick_start_with_dbeaver.html
+++ b/content/docs/latest/quick_start/quick_start_with_dbeaver.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. Getting Started With Kyuubi and DBeaver — Kyuubi 1.5.0-incubating documentation</title>
+ <title>3. Getting Started With Kyuubi and DBeaver — Kyuubi 1.5.1-incubating documentation</title>
@@ -328,22 +328,22 @@ By default, Kyuubi starts on port <code class="docutils literal notranslate"><sp
<div class="section" id="operations">
<h3><span class="section-number">3.4.2. </span>Operations<a class="headerlink" href="#operations" title="Permalink to this headline">¶</a></h3>
<p>Now, we can use the SQL editor to write queries to interact with Kyuubi server through the connection.</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">DESC</span> <span class="n">NAMESPACE</span> <span class="k">DEFAULT</span><span class="p">;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">DESC</span><span class="w"> </span><span class="n">NAMESPACE</span><span class="w"> </span><span class="k">DEFAULT</span><span class="p">;</span><span class="w"></span>
</pre></div>
</div>
<p><img alt="../_images/desc_database.png" src="../_images/desc_database.png" /></p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">spark_catalog</span><span class="p">.</span><span class="o">`</span><span class="k">default</span><span class="o">`</span><span class="p">.</span><span class="n">SRC</span><span class="p">(</span><span class="k">KEY</span> <span class="nb">INT</span><span class="p">,</span> <span class="n">VALUE</span> <span class="n">STRING</span><spa [...]
-<span class="k">INSERT</span> <span class="k">INTO</span> <span class="k">TABLE</span> <span class="n">spark_catalog</span><span class="p">.</span><span class="o">`</span><span class="k">default</span><span class="o">`</span><span class="p">.</span><span class="n">SRC</span> <span class="k">VALUES</span> <span class="p">(</span><span class="mi">11215016</span><span class="p">,</span> <span class="s1">'Kent Yao'</span><span class="p">);</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">spark_catalog</span><span class="p">.</span><span class="o">`</span><span class="k">default</span><span class="o">`</span><span class="p">.</span><span class="n">SRC</span><span class="p">(</span><span class="k">KEY</span><span class="w"> </span><span class="nb">INT</span><span class="p">,< [...]
+<span class="k">INSERT</span><span class="w"> </span><span class="k">INTO</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">spark_catalog</span><span class="p">.</span><span class="o">`</span><span class="k">default</span><span class="o">`</span><span class="p">.</span><span class="n">SRC</span><span class="w"> </span><span class="k">VALUES</span><span class="w"> </span><span class="p">(</span><span class="mi">11215016</span><span class="p" [...]
</pre></div>
</div>
<p><img alt="../_images/metadata.png" src="../_images/metadata.png" />
<em>Tips: zoom up if the pic looks small</em></p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SELECT</span> <span class="k">KEY</span> <span class="o">%</span> <span class="mi">10</span> <span class="k">AS</span> <span class="n">ID</span><span class="p">,</span> <span class="k">SUBSTRING</span><span class="p">(</span><span class="n">VALUE</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span> <span class="k">AS [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SELECT</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="o">%</span><span class="w"> </span><span class="mi">10</span><span class="w"> </span><span class="k">AS</span><span class="w"> </span><span class="n">ID</span><span class="p">,</span><span class="w"> </span><span class="k">SUBSTRING</span><span class="p">(</span><span class="n">VALUE</span><span [...]
</pre></div>
</div>
<p><img alt="../_images/query.png" src="../_images/query.png" />
<em>Tips: zoom up if the pic looks small</em></p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">DROP</span> <span class="k">TABLE</span> <span class="n">spark_catalog</span><span class="p">.</span><span class="o">`</span><span class="k">default</span><span class="o">`</span><span class="p">.</span><span class="n">SRC</span><span class="p">;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">DROP</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">spark_catalog</span><span class="p">.</span><span class="o">`</span><span class="k">default</span><span class="o">`</span><span class="p">.</span><span class="n">SRC</span><span class="p">;</span><span class="w"></span>
</pre></div>
</div>
</div>
@@ -357,55 +357,55 @@ By default, Kyuubi starts on port <code class="docutils literal notranslate"><sp
<p><img alt="../_images/viewdata.png" src="../_images/viewdata.png" />
<em>Tips: zoom up if the pic looks small</em></p>
<p>And we can write simple or complex SQL to manipulate data, for example, here is the query 41 generated by TPCDS <code class="docutils literal notranslate"><span class="pre">dsqgen</span></code> tool.</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SELECT</span> <span class="k">DISTINCT</span> <span class="p">(</span><span class="n">i_product_name</span><span class="p">)</span>
-<span class="k">FROM</span> <span class="n">item</span> <span class="n">i1</span>
-<span class="k">WHERE</span> <span class="n">i_manufact_id</span> <span class="k">BETWEEN</span> <span class="mi">738</span> <span class="k">AND</span> <span class="mi">738</span> <span class="o">+</span> <span class="mi">40</span>
- <span class="k">AND</span> <span class="p">(</span><span class="k">SELECT</span> <span class="k">count</span><span class="p">(</span><span class="o">*</span><span class="p">)</span> <span class="k">AS</span> <span class="n">item_cnt</span>
-<span class="k">FROM</span> <span class="n">item</span>
-<span class="k">WHERE</span> <span class="p">(</span><span class="n">i_manufact</span> <span class="o">=</span> <span class="n">i1</span><span class="p">.</span><span class="n">i_manufact</span> <span class="k">AND</span>
- <span class="p">((</span><span class="n">i_category</span> <span class="o">=</span> <span class="s1">'Women'</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_color</span> <span class="o">=</span> <span class="s1">'powder'</span> <span class="k">OR</span> <span class="n">i_color</span> <span class="o">=</span> <span class="s1">'khaki'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Ounce'</span> <span class="k">OR</span> <span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Oz'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_size</span> <span class="o">=</span> <span class="s1">'medium'</span> <span class="k">OR</span> <span class="n">i_size</span> <span class="o">=</span> <span class="s1">'extra large'</span><span class="p">)</span>
- <span class="p">)</span> <span class="k">OR</span>
- <span class="p">(</span><span class="n">i_category</span> <span class="o">=</span> <span class="s1">'Women'</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_color</span> <span class="o">=</span> <span class="s1">'brown'</span> <span class="k">OR</span> <span class="n">i_color</span> <span class="o">=</span> <span class="s1">'honeydew'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Bunch'</span> <span class="k">OR</span> <span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Ton'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_size</span> <span class="o">=</span> <span class="s1">'N/A'</span> <span class="k">OR</span> <span class="n">i_size</span> <span class="o">=</span> <span class="s1">'small'</span><span class="p">)</span>
- <span class="p">)</span> <span class="k">OR</span>
- <span class="p">(</span><span class="n">i_category</span> <span class="o">=</span> <span class="s1">'Men'</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_color</span> <span class="o">=</span> <span class="s1">'floral'</span> <span class="k">OR</span> <span class="n">i_color</span> <span class="o">=</span> <span class="s1">'deep'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_units</span> <span class="o">=</span> <span class="s1">'N/A'</span> <span class="k">OR</span> <span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Dozen'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_size</span> <span class="o">=</span> <span class="s1">'petite'</span> <span class="k">OR</span> <span class="n">i_size</span> <span class="o">=</span> <span class="s1">'large'</span><span class="p">)</span>
- <span class="p">)</span> <span class="k">OR</span>
- <span class="p">(</span><span class="n">i_category</span> <span class="o">=</span> <span class="s1">'Men'</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_color</span> <span class="o">=</span> <span class="s1">'light'</span> <span class="k">OR</span> <span class="n">i_color</span> <span class="o">=</span> <span class="s1">'cornflower'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Box'</span> <span class="k">OR</span> <span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Pound'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_size</span> <span class="o">=</span> <span class="s1">'medium'</span> <span class="k">OR</span> <span class="n">i_size</span> <span class="o">=</span> <span class="s1">'extra large'</span><span class="p">)</span>
- <span class="p">)))</span> <span class="k">OR</span>
- <span class="p">(</span><span class="n">i_manufact</span> <span class="o">=</span> <span class="n">i1</span><span class="p">.</span><span class="n">i_manufact</span> <span class="k">AND</span>
- <span class="p">((</span><span class="n">i_category</span> <span class="o">=</span> <span class="s1">'Women'</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_color</span> <span class="o">=</span> <span class="s1">'midnight'</span> <span class="k">OR</span> <span class="n">i_color</span> <span class="o">=</span> <span class="s1">'snow'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Pallet'</span> <span class="k">OR</span> <span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Gross'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_size</span> <span class="o">=</span> <span class="s1">'medium'</span> <span class="k">OR</span> <span class="n">i_size</span> <span class="o">=</span> <span class="s1">'extra large'</span><span class="p">)</span>
- <span class="p">)</span> <span class="k">OR</span>
- <span class="p">(</span><span class="n">i_category</span> <span class="o">=</span> <span class="s1">'Women'</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_color</span> <span class="o">=</span> <span class="s1">'cyan'</span> <span class="k">OR</span> <span class="n">i_color</span> <span class="o">=</span> <span class="s1">'papaya'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Cup'</span> <span class="k">OR</span> <span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Dram'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_size</span> <span class="o">=</span> <span class="s1">'N/A'</span> <span class="k">OR</span> <span class="n">i_size</span> <span class="o">=</span> <span class="s1">'small'</span><span class="p">)</span>
- <span class="p">)</span> <span class="k">OR</span>
- <span class="p">(</span><span class="n">i_category</span> <span class="o">=</span> <span class="s1">'Men'</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_color</span> <span class="o">=</span> <span class="s1">'orange'</span> <span class="k">OR</span> <span class="n">i_color</span> <span class="o">=</span> <span class="s1">'frosted'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Each'</span> <span class="k">OR</span> <span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Tbl'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_size</span> <span class="o">=</span> <span class="s1">'petite'</span> <span class="k">OR</span> <span class="n">i_size</span> <span class="o">=</span> <span class="s1">'large'</span><span class="p">)</span>
- <span class="p">)</span> <span class="k">OR</span>
- <span class="p">(</span><span class="n">i_category</span> <span class="o">=</span> <span class="s1">'Men'</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_color</span> <span class="o">=</span> <span class="s1">'forest'</span> <span class="k">OR</span> <span class="n">i_color</span> <span class="o">=</span> <span class="s1">'ghost'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Lb'</span> <span class="k">OR</span> <span class="n">i_units</span> <span class="o">=</span> <span class="s1">'Bundle'</span><span class="p">)</span> <span class="k">AND</span>
- <span class="p">(</span><span class="n">i_size</span> <span class="o">=</span> <span class="s1">'medium'</span> <span class="k">OR</span> <span class="n">i_size</span> <span class="o">=</span> <span class="s1">'extra large'</span><span class="p">)</span>
- <span class="p">))))</span> <span class="o">></span> <span class="mi">0</span>
-<span class="k">ORDER</span> <span class="k">BY</span> <span class="n">i_product_name</span>
-<span class="k">LIMIT</span> <span class="mi">100</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">SELECT</span><span class="w"> </span><span class="k">DISTINCT</span><span class="w"> </span><span class="p">(</span><span class="n">i_product_name</span><span class="p">)</span><span class="w"></span>
+<span class="k">FROM</span><span class="w"> </span><span class="n">item</span><span class="w"> </span><span class="n">i1</span><span class="w"></span>
+<span class="k">WHERE</span><span class="w"> </span><span class="n">i_manufact_id</span><span class="w"> </span><span class="k">BETWEEN</span><span class="w"> </span><span class="mi">738</span><span class="w"> </span><span class="k">AND</span><span class="w"> </span><span class="mi">738</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">40</span><span class="w"></span>
+<span class="w"> </span><span class="k">AND</span><span class="w"> </span><span class="p">(</span><span class="k">SELECT</span><span class="w"> </span><span class="k">count</span><span class="p">(</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="k">AS</span><span class="w"> </span><span class="n">item_cnt</span><span class="w"></span>
+<span class="k">FROM</span><span class="w"> </span><span class="n">item</span><span class="w"></span>
+<span class="k">WHERE</span><span class="w"> </span><span class="p">(</span><span class="n">i_manufact</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">i1</span><span class="p">.</span><span class="n">i_manufact</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">((</span><span class="n">i_category</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Women'</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'powder'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'khaki'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span class [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Ounce'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Oz'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span class="w" [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'medium'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'extra large'</span><span class="p">)</span><span class="w"></span>
+<span class="w"> </span><span class="p">)</span><span class="w"> </span><span class="k">OR</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_category</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Women'</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'brown'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'honeydew'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span c [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Bunch'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Ton'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span class= [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'N/A'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'small'</span><span class="p">)</span><span class="w"></span>
+<span class="w"> </span><span class="p">)</span><span class="w"> </span><span class="k">OR</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_category</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Men'</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'floral'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'deep'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span clas [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'N/A'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Dozen'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span class= [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'petite'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'large'</span><span class="p">)</span><span class="w"></span>
+<span class="w"> </span><span class="p">)</span><span class="w"> </span><span class="k">OR</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_category</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Men'</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'light'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'cornflower'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Box'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Pound'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span class= [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'medium'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'extra large'</span><span class="p">)</span><span class="w"></span>
+<span class="w"> </span><span class="p">)))</span><span class="w"> </span><span class="k">OR</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_manufact</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">i1</span><span class="p">.</span><span class="n">i_manufact</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">((</span><span class="n">i_category</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Women'</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'midnight'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'snow'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span cl [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Pallet'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Gross'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span cla [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'medium'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'extra large'</span><span class="p">)</span><span class="w"></span>
+<span class="w"> </span><span class="p">)</span><span class="w"> </span><span class="k">OR</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_category</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Women'</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'cyan'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'papaya'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span cl [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Cup'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Dram'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span class [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'N/A'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'small'</span><span class="p">)</span><span class="w"></span>
+<span class="w"> </span><span class="p">)</span><span class="w"> </span><span class="k">OR</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_category</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Men'</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'orange'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'frosted'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Each'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Tbl'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span class [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'petite'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'large'</span><span class="p">)</span><span class="w"></span>
+<span class="w"> </span><span class="p">)</span><span class="w"> </span><span class="k">OR</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_category</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Men'</span><span class="w"> </span><span class="k">AND</span><span class="w"></span>
+<span class="w"> </span><span class="p">(</span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'forest'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_color</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'ghost'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span c [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Lb'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'Bundle'</span><span class="p">)</span><span class="w"> </span><span class="k">AND</span><span clas [...]
+<span class="w"> </span><span class="p">(</span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'medium'</span><span class="w"> </span><span class="k">OR</span><span class="w"> </span><span class="n">i_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'extra large'</span><span class="p">)</span><span class="w"></span>
+<span class="w"> </span><span class="p">))))</span><span class="w"> </span><span class="o">></span><span class="w"> </span><span class="mi">0</span><span class="w"></span>
+<span class="k">ORDER</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">i_product_name</span><span class="w"></span>
+<span class="k">LIMIT</span><span class="w"> </span><span class="mi">100</span><span class="w"></span>
</pre></div>
</div>
<p><img alt="../_images/query41_result.png" src="../_images/query41_result.png" />
diff --git a/content/docs/latest/quick_start/quick_start_with_helm.html b/content/docs/latest/quick_start/quick_start_with_helm.html
index 1a990a4..19c9d10 100644
--- a/content/docs/latest/quick_start/quick_start_with_helm.html
+++ b/content/docs/latest/quick_start/quick_start_with_helm.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Getting Started With Kyuubi on kubernetes — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Getting Started With Kyuubi on kubernetes — Kyuubi 1.5.1-incubating documentation</title>
@@ -245,7 +245,7 @@ REVISION: <span class="m">1</span>
TEST SUITE: None
NOTES:
Get kyuubi expose URL by running these commands:
- <span class="nb">export</span> <span class="nv">NODE_PORT</span><span class="o">=</span><span class="k">$(</span>kubectl get --namespace kyuubi -o <span class="nv">jsonpath</span><span class="o">=</span><span class="s2">"{.spec.ports[0].nodePort}"</span> services kyuubi-helm-nodeport<span class="k">)</span>
+ <span class="nb">export</span> <span class="nv">NODE_PORT</span><span class="o">=</span><span class="k">$(</span>kubectl get --namespace kyuubi -o <span class="nv">jsonpath</span><span class="o">=</span><span class="s2">"{.spec.ports[0].nodePort}"</span> services kyuubi-svc<span class="k">)</span>
<span class="nb">export</span> <span class="nv">NODE_IP</span><span class="o">=</span><span class="k">$(</span>kubectl get nodes --namespace kyuubi -o <span class="nv">jsonpath</span><span class="o">=</span><span class="s2">"{.items[0].status.addresses[0].address}"</span><span class="k">)</span>
<span class="nb">echo</span> <span class="nv">$NODE_IP</span>:<span class="nv">$NODE_PORT</span>
</pre></div>
@@ -264,29 +264,29 @@ Get kyuubi expose URL by running these commands:
<div class="section" id="edit-server-config">
<h4>Edit server config<a class="headerlink" href="#edit-server-config" title="Permalink to this headline">¶</a></h4>
<p>Modify <code class="docutils literal notranslate"><span class="pre">values.yaml</span></code> under <code class="docutils literal notranslate"><span class="pre">${KYUUBI_HOME}/docker/helm</span></code>:</p>
-<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="c1"># Kyuubi server numbers</span>
-<span class="nt">replicaCount</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">2</span>
-
-<span class="nt">image</span><span class="p">:</span>
- <span class="nt">repository</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">apache/kyuubi</span>
- <span class="nt">pullPolicy</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">Always</span>
- <span class="c1"># Overrides the image tag whose default is the chart appVersion.</span>
- <span class="nt">tag</span><span class="p">:</span> <span class="s">"master-snapshot"</span>
-
-<span class="nt">server</span><span class="p">:</span>
- <span class="nt">bind</span><span class="p">:</span>
- <span class="nt">host</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
- <span class="nt">port</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">10009</span>
- <span class="nt">conf</span><span class="p">:</span>
- <span class="nt">mountPath</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">/opt/kyuubi/conf</span>
-
-<span class="nt">service</span><span class="p">:</span>
- <span class="nt">type</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">NodePort</span>
- <span class="c1"># The default port limit of kubernetes is 30000-32767</span>
- <span class="c1"># to change:</span>
- <span class="c1"># vim kube-apiserver.yaml (usually under path: /etc/kubernetes/manifests/)</span>
- <span class="c1"># add or change line 'service-node-port-range=1-32767' under kube-apiserver</span>
- <span class="nt">port</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">30009</span>
+<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="c1"># Kyuubi server numbers</span><span class="w"></span>
+<span class="nt">replicaCount</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">2</span><span class="w"></span>
+
+<span class="nt">image</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="nt">repository</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">apache/kyuubi</span><span class="w"></span>
+<span class="w"> </span><span class="nt">pullPolicy</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Always</span><span class="w"></span>
+<span class="w"> </span><span class="c1"># Overrides the image tag whose default is the chart appVersion.</span><span class="w"></span>
+<span class="w"> </span><span class="nt">tag</span><span class="p">:</span><span class="w"> </span><span class="s">"master-snapshot"</span><span class="w"></span>
+
+<span class="nt">server</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="nt">bind</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="nt">host</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span><span class="w"></span>
+<span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10009</span><span class="w"></span>
+<span class="w"> </span><span class="nt">conf</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="nt">mountPath</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/opt/kyuubi/conf</span><span class="w"></span>
+
+<span class="nt">service</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">NodePort</span><span class="w"></span>
+<span class="w"> </span><span class="c1"># The default port limit of kubernetes is 30000-32767</span><span class="w"></span>
+<span class="w"> </span><span class="c1"># to change:</span><span class="w"></span>
+<span class="w"> </span><span class="c1"># vim kube-apiserver.yaml (usually under path: /etc/kubernetes/manifests/)</span><span class="w"></span>
+<span class="w"> </span><span class="c1"># add or change line 'service-node-port-range=1-32767' under kube-apiserver</span><span class="w"></span>
+<span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30009</span><span class="w"></span>
</pre></div>
</div>
</div>
diff --git a/content/docs/latest/quick_start/quick_start_with_hue.html b/content/docs/latest/quick_start/quick_start_with_hue.html
index 33eef88..49aa932 100644
--- a/content/docs/latest/quick_start/quick_start_with_hue.html
+++ b/content/docs/latest/quick_start/quick_start_with_hue.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>7. Getting Started with Kyuubi and Cloudera Hue — Kyuubi 1.5.0-incubating documentation</title>
+ <title>7. Getting Started with Kyuubi and Cloudera Hue — Kyuubi 1.5.1-incubating documentation</title>
@@ -261,28 +261,28 @@ there are several known limitations of network, and you can find
<ol class="simple">
<li><p>Modify the <code class="docutils literal notranslate"><span class="pre">hue.ini</span></code></p></li>
</ol>
-<div class="highlight-ini notranslate"><div class="highlight"><pre><span></span><span class="k">[beeswax]</span>
- <span class="c1"># Kyuubi 1.1.x support thrift version from 1 to 10</span>
- <span class="na">thrift_version</span><span class="o">=</span><span class="s">7</span>
- <span class="c1"># change to your username to avoid permissions issue for local test</span>
- <span class="na">auth_username</span><span class="o">=</span><span class="s">chengpan</span>
-
-<span class="k">[notebook]</span>
- <span class="k">[[interpreters]]</span>
- <span class="k">[[[sql]]]</span>
- <span class="na">name</span><span class="o">=</span><span class="s">SparkSQL</span>
- <span class="na">interface</span><span class="o">=</span><span class="s">hiveserver2</span>
-
-<span class="k">[spark]</span>
- <span class="c1"># Host of the Spark Thrift Server</span>
- <span class="c1"># For macOS users, use docker.for.mac.host.internal to access host network</span>
- <span class="na">sql_server_host</span><span class="o">=</span><span class="s">docker.for.mac.host.internal</span>
-
- <span class="c1"># Port of the Spark Thrift Server</span>
- <span class="na">sql_server_port</span><span class="o">=</span><span class="s">10009</span>
-
-<span class="c1"># other configurations</span>
-<span class="na">...</span>
+<div class="highlight-ini notranslate"><div class="highlight"><pre><span></span><span class="k">[beeswax]</span><span class="w"></span>
+<span class="w"> </span><span class="c1"># Kyuubi 1.1.x support thrift version from 1 to 10</span><span class="w"></span>
+<span class="w"> </span><span class="na">thrift_version</span><span class="o">=</span><span class="s">7</span><span class="w"></span>
+<span class="w"> </span><span class="c1"># change to your username to avoid permissions issue for local test</span><span class="w"></span>
+<span class="w"> </span><span class="na">auth_username</span><span class="o">=</span><span class="s">chengpan</span><span class="w"></span>
+
+<span class="k">[notebook]</span><span class="w"></span>
+<span class="w"> </span><span class="k">[[interpreters]]</span><span class="w"></span>
+<span class="w"> </span><span class="k">[[[sql]]]</span><span class="w"></span>
+<span class="w"> </span><span class="na">name</span><span class="o">=</span><span class="s">SparkSQL</span><span class="w"></span>
+<span class="w"> </span><span class="na">interface</span><span class="o">=</span><span class="s">hiveserver2</span><span class="w"></span>
+<span class="w"> </span>
+<span class="k">[spark]</span><span class="w"></span>
+<span class="w"> </span><span class="c1"># Host of the Spark Thrift Server</span><span class="w"></span>
+<span class="w"> </span><span class="c1"># For macOS users, use docker.for.mac.host.internal to access host network</span><span class="w"></span>
+<span class="w"> </span><span class="na">sql_server_host</span><span class="o">=</span><span class="s">docker.for.mac.host.internal</span><span class="w"></span>
+
+<span class="w"> </span><span class="c1"># Port of the Spark Thrift Server</span><span class="w"></span>
+<span class="w"> </span><span class="na">sql_server_port</span><span class="o">=</span><span class="s">10009</span><span class="w"></span>
+<span class="w"> </span>
+<span class="c1"># other configurations</span><span class="w"></span>
+<span class="na">...</span><span class="w"></span>
</pre></div>
</div>
</div>
diff --git a/content/docs/latest/quick_start/quick_start_with_jdbc.html b/content/docs/latest/quick_start/quick_start_with_jdbc.html
index 10415f8..fc1ace0 100644
--- a/content/docs/latest/quick_start/quick_start_with_jdbc.html
+++ b/content/docs/latest/quick_start/quick_start_with_jdbc.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>5. Getting Started With Hive JDBC — Kyuubi 1.5.0-incubating documentation</title>
+ <title>5. Getting Started With Hive JDBC — Kyuubi 1.5.1-incubating documentation</title>
@@ -227,7 +227,7 @@
</pre></div>
</div>
<p>You can add below dependency to your <code class="docutils literal notranslate"><span class="pre">pom.xml</span></code> file in your application.</p>
-<div class="highlight-xml notranslate"><div class="highlight"><pre><span></span><span class="c"><!-- https://mvnrepository.com/artifact/org.apache.hive/hive-jdbc --></span>
+<div class="highlight-xml notranslate"><div class="highlight"><pre><span></span><span class="cm"><!-- https://mvnrepository.com/artifact/org.apache.hive/hive-jdbc --></span>
<span class="nt"><dependency></span>
<span class="nt"><groupId></span>org.apache.hive<span class="nt"></groupId></span>
<span class="nt"><artifactId></span>hive-jdbc<span class="nt"></artifactId></span>
@@ -236,7 +236,7 @@
<span class="nt"><dependency></span>
<span class="nt"><groupId></span>org.apache.hadoop<span class="nt"></groupId></span>
<span class="nt"><artifactId></span>hadoop-common<span class="nt"></artifactId></span>
- <span class="c"><!-- keep consistent with the build hadoop version --></span>
+ <span class="cm"><!-- keep consistent with the build hadoop version --></span>
<span class="nt"><version></span>2.7.4<span class="nt"></version></span>
<span class="nt"></dependency></span>
</pre></div>
diff --git a/content/docs/latest/quick_start/quick_start_with_jupyter.html b/content/docs/latest/quick_start/quick_start_with_jupyter.html
index 27991ae..535eced 100644
--- a/content/docs/latest/quick_start/quick_start_with_jupyter.html
+++ b/content/docs/latest/quick_start/quick_start_with_jupyter.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Getting Started With Hive Jupyter Lap — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Getting Started With Hive Jupyter Lap — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/search.html b/content/docs/latest/search.html
index 420547b..863ddda 100644
--- a/content/docs/latest/search.html
+++ b/content/docs/latest/search.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Search — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Search — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/searchindex.js b/content/docs/latest/searchindex.js
index 9d96b5c..dea6dff 100644
--- a/content/docs/latest/searchindex.js
+++ b/content/docs/latest/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["appendix/index","appendix/terminology","client/hive_jdbc","client/index","client/kerberized_kyuubi","community/CONTRIBUTING","community/collaborators","community/index","community/release","deployment/engine_lifecycle","deployment/engine_on_kubernetes","deployment/engine_on_yarn","deployment/engine_share_level","deployment/high_availability_guide","deployment/hive_metastore","deployment/index","deployment/kyuubi_on_kubernetes","deployment/settings","deployment [...]
\ No newline at end of file
+Search.setIndex({docnames:["appendix/index","appendix/terminology","client/hive_jdbc","client/index","client/kerberized_kyuubi","community/CONTRIBUTING","community/collaborators","community/index","community/release","deployment/engine_lifecycle","deployment/engine_on_kubernetes","deployment/engine_on_yarn","deployment/engine_share_level","deployment/high_availability_guide","deployment/hive_metastore","deployment/index","deployment/kyuubi_on_kubernetes","deployment/settings","deployment [...]
\ No newline at end of file
diff --git a/content/docs/latest/security/authentication.html b/content/docs/latest/security/authentication.html
index 2a82e21..34e2840 100644
--- a/content/docs/latest/security/authentication.html
+++ b/content/docs/latest/security/authentication.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Kyuubi Authentication Mechanism — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Kyuubi Authentication Mechanism — Kyuubi 1.5.1-incubating documentation</title>
@@ -227,7 +227,7 @@ For other secured services that this user wants to interact with, he/she also ne
</thead>
<tbody>
<tr>
-<td>kyuubi.authentication</td>
+<td><code>kyuubi.authentication</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'>NONE</div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>Client authentication types.<ul> <li>NOSASL: raw transport.</li> <li>NONE: no authentication check.</li> <li>KERBEROS: Kerberos/GSSAPI authentication.</li> <li>LDAP: Lightweight Directory Access Protocol authentication.</li></ul></div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
@@ -244,31 +244,31 @@ For other secured services that this user wants to interact with, he/she also ne
</thead>
<tbody>
<tr>
-<td>kyuubi.authentication</td>
+<td><code>kyuubi.authentication</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'>NONE</div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>Client authentication types.<ul> <li>NOSASL: raw transport.</li> <li>NONE: no authentication check.</li> <li>KERBEROS: Kerberos/GSSAPI authentication.</li> <li>LDAP: Lightweight Directory Access Protocol authentication.</li></ul></div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.authentication<br>.ldap.base.dn</td>
+<td><code>kyuubi.authentication.ldap.base.dn</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'><undefined></div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>LDAP base DN.</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.authentication<br>.ldap.domain</td>
+<td><code>kyuubi.authentication.ldap.domain</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'><undefined></div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>LDAP base DN.</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.authentication<br>.ldap.url</td>
+<td><code>kyuubi.authentication.ldap.url</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'><undefined></div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>SPACE character separated LDAP connection URL(s).</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.authentication<br>.sasl.qop</td>
+<td><code>kyuubi.authentication.sasl.qop</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'>auth</div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>Sasl QOP enable higher levels of protection for Kyuubi communication with clients.<ul> <li>auth - authentication only (default)</li> <li>auth-int - authentication plus integrity protection</li> <li>auth-conf - authentication plus integrity and confidentiality protection. This is applicable only if Kyuubi is configured to use Kerberos authentication.</li> </ul></div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
@@ -295,25 +295,25 @@ each principal shares a secret, such as a password, with the KDC.</p>
</thead>
<tbody>
<tr>
-<td>kyuubi.kinit<br>.principal</td>
+<td><code>kyuubi.kinit.principal</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'><undefined></div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>Name of the Kerberos principal.</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.kinit.keytab</td>
+<td><code>kyuubi.kinit.keytab</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'><undefined></div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>Location of Kyuubi server's keytab.</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.kinit.interval</td>
+<td><code>kyuubi.kinit.interval</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'>PT1H</div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>How often will Kyuubi server run <code>kinit -kt [keytab] [principal]</code> to renew the local Kerberos credentials cache</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.kinit.max<br>.attempts</td>
+<td><code>kyuubi.kinit.max.attempts</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'>10</div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>How many times will <code>kinit</code> process retry</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
diff --git a/content/docs/latest/security/authorization.html b/content/docs/latest/security/authorization.html
index 01af1c7..2f31e2e 100644
--- a/content/docs/latest/security/authorization.html
+++ b/content/docs/latest/security/authorization.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>4. ACL Management Guide — Kyuubi 1.5.0-incubating documentation</title>
+ <title>4. ACL Management Guide — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/security/hadoop_credentials_manager.html b/content/docs/latest/security/hadoop_credentials_manager.html
index 5cc23fa..5c110ca 100644
--- a/content/docs/latest/security/hadoop_credentials_manager.html
+++ b/content/docs/latest/security/hadoop_credentials_manager.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. Hadoop Credentials Manager — Kyuubi 1.5.0-incubating documentation</title>
+ <title>3. Hadoop Credentials Manager — Kyuubi 1.5.1-incubating documentation</title>
@@ -101,6 +101,7 @@
<li class="toctree-l3"><a class="reference internal" href="#configurations">3.1. Configurations</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#cluster-services">3.1.1. Cluster Services</a></li>
<li class="toctree-l4"><a class="reference internal" href="#credentials-renewal">3.1.2. Credentials Renewal</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#required-security-configs">3.1.3. Required Security Configs</a></li>
</ul>
</li>
</ul>
@@ -271,35 +272,35 @@ Hadoop Credentials Manager is initialized when Kyuubi server starts.</p>
</thead>
<tbody>
<tr>
-<td>kyuubi.credentials<br>.hadoopfs.enabled</td>
+<td><code>kyuubi.credentials.hadoopfs.enabled</code></td>
<td><div style='width: 65pt;word-wrap: break-word;white-space: normal'>true</div></td>
<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>Whether to renew Hadoop filesystem delegation tokens</div></td>
<td><div style='width: 30pt'>boolean</div></td>
<td><div style='width: 20pt'>1.4.0</div></td>
</tr>
<tr>
-<td>kyuubi.credentials<br>.hadoopfs.uris</td>
+<td><code>kyuubi.credentials.hadoopfs.uris</code></td>
<td><div style='width: 65pt;word-wrap: break-word;white-space: normal'></div></td>
<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>Extra Hadoop filesystem URIs for which to request delegation tokens. The filesystem that hosts fs.defaultFS does not need to be listed here.</div></td>
<td><div style='width: 30pt'>seq</div></td>
<td><div style='width: 20pt'>1.4.0</div></td>
</tr>
<tr>
-<td>kyuubi.credentials<br>.hive.enabled</td>
+<td><code>kyuubi.credentials.hive.enabled</code></td>
<td><div style='width: 65pt;word-wrap: break-word;white-space: normal'>true</div></td>
<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>Whether to renew Hive metastore delegation token</div></td>
<td><div style='width: 30pt'>boolean</div></td>
<td><div style='width: 20pt'>1.4.0</div></td>
</tr>
<tr>
-<td>kyuubi.credentials<br>.renewal.interval</td>
+<td><code>kyuubi.credentials.renewal.interval</code></td>
<td><div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT1H</div></td>
<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>How often Kyuubi renews one user's delegation tokens</div></td>
<td><div style='width: 30pt'>duration</div></td>
<td><div style='width: 20pt'>1.4.0</div></td>
</tr>
<tr>
-<td>kyuubi.credentials<br>.renewal.retry.wait</td>
+<td><code>kyuubi.credentials.renewal.retry.wait</code></td>
<td><div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT1M</div></td>
<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>How long to wait before retrying to fetch new credentials after a failure.</div></td>
<td><div style='width: 30pt'>duration</div></td>
@@ -307,6 +308,45 @@ Hadoop Credentials Manager is initialized when Kyuubi server starts.</p>
</tr>
</tbody>
</table></div>
+<div class="section" id="required-security-configs">
+<h3><span class="section-number">3.1.3. </span>Required Security Configs<a class="headerlink" href="#required-security-configs" title="Permalink to this headline">¶</a></h3>
+<p>The necessary configurations for hdfs and hive to obtain delegation token are as follows:</p>
+<table border="1" class="docutils">
+<thead>
+<tr>
+<th>Key</th>
+<th>Meaning</th>
+<th>value</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>hadoop.security.authentication</code></td>
+<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>Set the authentication for the cluster</div></td>
+<td><div style='width: 120pt;word-wrap: break-word;white-space: normal'>kerberos</div></td>
+</tr>
+<tr>
+<td><code>hive.metastore.uris</code></td>
+<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>URI for client to contact metastore server</div></td>
+<td><div style='width: 120pt;word-wrap: break-word;white-space: normal'>thrift://{metastoreHost}:{metastorePort}}</div></td>
+</tr>
+<tr>
+<td><code>hive.metastore.sasl.enabled</code></td>
+<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>If true, the metastore thrift interface will be secured with SASL.Clients must authenticate with Kerberos.</div></td>
+<td><div style='width: 120pt;word-wrap: break-word;white-space: normal'>true</div></td>
+</tr>
+<tr>
+<td><code>hive.metastore.kerberos.principal</code></td>
+<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>The service principal for the metastore thrift server. The special string _HOST will be replaced automatically with the correct host name.</div></td>
+<td><div style='width: 120pt;word-wrap: break-word;white-space: normal'>for example hive/_HOST@${realm}</div></td>
+</tr>
+<tr>
+<td><code>hive.metastore.kerberos.keytab.file</code></td>
+<td><div style='width: 170pt;word-wrap: break-word;white-space: normal'>The path to the Kerberos Keytab file containing the metastore thrift server's service principal.</div></td>
+<td><div style='width: 120pt;word-wrap: break-word;white-space: normal'>for example /etc/security/keytabs/hive.service.keytab</div></td>
+</tr>
+</tbody>
+</table></div>
</div>
</div>
diff --git a/content/docs/latest/security/index.html b/content/docs/latest/security/index.html
index 975a87c..2a0cb27 100644
--- a/content/docs/latest/security/index.html
+++ b/content/docs/latest/security/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Kyuubi Security Overview — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Kyuubi Security Overview — Kyuubi 1.5.1-incubating documentation</title>
@@ -42,7 +42,7 @@
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="1. Kyuubi Authentication Mechanism" href="authentication.html" />
- <link rel="prev" title="3. Solution for Big Result Sets" href="../deployment/spark/incremental_collection.html" />
+ <link rel="prev" title="5.3. Solution for Big Result Sets" href="../deployment/spark/incremental_collection.html" />
</head>
<body class="wy-body-for-nav">
@@ -218,7 +218,7 @@
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="authentication.html" class="btn btn-neutral float-right" title="1. Kyuubi Authentication Mechanism" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
- <a href="../deployment/spark/incremental_collection.html" class="btn btn-neutral float-left" title="3. Solution for Big Result Sets" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+ <a href="../deployment/spark/incremental_collection.html" class="btn btn-neutral float-left" title="5.3. Solution for Big Result Sets" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
</div>
<hr/>
diff --git a/content/docs/latest/security/kinit.html b/content/docs/latest/security/kinit.html
index e762d69..cc19651 100644
--- a/content/docs/latest/security/kinit.html
+++ b/content/docs/latest/security/kinit.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Kinit Auxiliary Service — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Kinit Auxiliary Service — Kyuubi 1.5.1-incubating documentation</title>
@@ -267,25 +267,25 @@ For example, <code class="docutils literal notranslate"><span class="pre">/tmp/k
</thead>
<tbody>
<tr>
-<td>kyuubi.kinit<br>.principal</td>
+<td><code>kyuubi.kinit.principal</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'><undefined></div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>Name of the Kerberos principal.</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.kinit.keytab</td>
+<td><code>kyuubi.kinit.keytab</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'><undefined></div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>Location of Kyuubi server's keytab.</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.kinit.interval</td>
+<td><code>kyuubi.kinit.interval</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'>PT1H</div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>How often will Kyuubi server run <code>kinit -kt [keytab] [principal]</code> to renew the local Kerberos credentials cache</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
</tr>
<tr>
-<td>kyuubi.kinit.max<br>.attempts</td>
+<td><code>kyuubi.kinit.max.attempts</code></td>
<td><div style='width: 80pt;word-wrap: break-word;white-space: normal'>10</div></td>
<td><div style='width: 200pt;word-wrap: break-word;white-space: normal'>How many times will <code>kinit</code> process retry</div></td>
<td><div style='width: 20pt'>1.0.0</div></td>
diff --git a/content/docs/latest/sql/functions.html b/content/docs/latest/sql/functions.html
index 9271e88..52a88c0 100644
--- a/content/docs/latest/sql/functions.html
+++ b/content/docs/latest/sql/functions.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>2. Auxiliary SQL Functions for Spark SQL — Kyuubi 1.5.0-incubating documentation</title>
+ <title>2. Auxiliary SQL Functions for Spark SQL — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/sql/index.html b/content/docs/latest/sql/index.html
index 211f6f5..3a3f2b0 100644
--- a/content/docs/latest/sql/index.html
+++ b/content/docs/latest/sql/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>SQL References — Kyuubi 1.5.0-incubating documentation</title>
+ <title>SQL References — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/sql/rules.html b/content/docs/latest/sql/rules.html
index ff2f478..49eb457 100644
--- a/content/docs/latest/sql/rules.html
+++ b/content/docs/latest/sql/rules.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Auxiliary SQL extension for Spark SQL — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Auxiliary SQL extension for Spark SQL — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/sql/z-order-benchmark.html b/content/docs/latest/sql/z-order-benchmark.html
index 2225f95..2991200 100644
--- a/content/docs/latest/sql/z-order-benchmark.html
+++ b/content/docs/latest/sql/z-order-benchmark.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>4. Z-order Benchmark — Kyuubi 1.5.0-incubating documentation</title>
+ <title>4. Z-order Benchmark — Kyuubi 1.5.1-incubating documentation</title>
@@ -329,11 +329,11 @@ spark.stop<span class="o">()</span>
</pre></div>
</div>
<p>Z-order Optimize statement:</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">set</span> <span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">hive</span><span class="p">.</span><span class="n">convertMetastoreParquet</span><span class="o">=</span><span class="k">false</span><span class="p">;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">set</span><span class="w"> </span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">hive</span><span class="p">.</span><span class="n">convertMetastoreParquet</span><span class="o">=</span><span class="k">false</span><span class="p">;</span><span class="w"></span>
-<span class="n">OPTIMIZE</span> <span class="n">conn_zorder_only_ip</span> <span class="n">ZORDER</span> <span class="k">BY</span> <span class="n">src_ip</span><span class="p">,</span> <span class="n">dst_ip</span><span class="p">;</span>
+<span class="n">OPTIMIZE</span><span class="w"> </span><span class="n">conn_zorder_only_ip</span><span class="w"> </span><span class="n">ZORDER</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">src_ip</span><span class="p">,</span><span class="w"> </span><span class="n">dst_ip</span><span class="p">;</span><span class="w"></span>
-<span class="n">OPTIMIZE</span> <span class="n">zorder_test</span><span class="p">.</span><span class="n">conn_zorder</span> <span class="n">ZORDER</span> <span class="k">BY</span> <span class="n">src_ip</span><span class="p">,</span> <span class="n">src_port</span><span class="p">,</span> <span class="n">dst_ip</span><span class="p">,</span> <span class="n">dst_port</span><span class="p">;</span>
+<span class="n">OPTIMIZE</span><span class="w"> </span><span class="n">zorder_test</span><span class="p">.</span><span class="n">conn_zorder</span><span class="w"> </span><span class="n">ZORDER</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">src_ip</span><span class="p">,</span><span class="w"> </span><span class="n">src_port</span><span class="p">,</span><span class="w"> </span><span class="n">dst_ip</span><span class="p">,</span><span clas [...]
</pre></div>
</div>
<p>ORDER BY statement:</p>
@@ -343,13 +343,13 @@ spark.stop<span class="o">()</span>
</pre></div>
</div>
<p>Query statement:</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">set</span> <span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">hive</span><span class="p">.</span><span class="n">convertMetastoreParquet</span><span class="o">=</span><span class="k">true</span><span class="p">;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">set</span><span class="w"> </span><span class="n">spark</span><span class="p">.</span><span class="k">sql</span><span class="p">.</span><span class="n">hive</span><span class="p">.</span><span class="n">convertMetastoreParquet</span><span class="o">=</span><span class="k">true</span><span class="p">;</span><span class="w"></span>
-<span class="k">select</span> <span class="k">count</span><span class="p">(</span><span class="o">*</span><span class="p">)</span> <span class="k">from</span> <span class="n">conn_random_parquet</span> <span class="k">where</span> <span class="n">src_ip</span> <span class="k">like</span> <span class="s1">'157%'</span> <span class="k">and</span> <span class="n">dst_ip</span> <span class="k">like</span> <span class="s1">'216.%'</span><span class="p">;</span>
+<span class="k">select</span><span class="w"> </span><span class="k">count</span><span class="p">(</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="k">from</span><span class="w"> </span><span class="n">conn_random_parquet</span><span class="w"> </span><span class="k">where</span><span class="w"> </span><span class="n">src_ip</span><span class="w"> </span><span class="k">like</span><span class="w"> </span><span class="s1">'157%'</span><spa [...]
-<span class="k">select</span> <span class="k">count</span><span class="p">(</span><span class="o">*</span><span class="p">)</span> <span class="k">from</span> <span class="n">conn_zorder_only_ip</span> <span class="k">where</span> <span class="n">src_ip</span> <span class="k">like</span> <span class="s1">'157%'</span> <span class="k">and</span> <span class="n">dst_ip</span> <span class="k">like</span> <span class="s1">'216.%'</span><span class="p">;</span>
+<span class="k">select</span><span class="w"> </span><span class="k">count</span><span class="p">(</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="k">from</span><span class="w"> </span><span class="n">conn_zorder_only_ip</span><span class="w"> </span><span class="k">where</span><span class="w"> </span><span class="n">src_ip</span><span class="w"> </span><span class="k">like</span><span class="w"> </span><span class="s1">'157%'</span><spa [...]
-<span class="k">select</span> <span class="k">count</span><span class="p">(</span><span class="o">*</span><span class="p">)</span> <span class="k">from</span> <span class="n">conn_zorder</span> <span class="k">where</span> <span class="n">src_ip</span> <span class="k">like</span> <span class="s1">'157%'</span> <span class="k">and</span> <span class="n">dst_ip</span> <span class="k">like</span> <span class="s1">'216.%'</span><span class="p">;</span>
+<span class="k">select</span><span class="w"> </span><span class="k">count</span><span class="p">(</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="k">from</span><span class="w"> </span><span class="n">conn_zorder</span><span class="w"> </span><span class="k">where</span><span class="w"> </span><span class="n">src_ip</span><span class="w"> </span><span class="k">like</span><span class="w"> </span><span class="s1">'157%'</span><span class= [...]
</pre></div>
</div>
<div class="section" id="benchmark-result">
diff --git a/content/docs/latest/sql/z-order-introduction.html b/content/docs/latest/sql/z-order-introduction.html
index bfcb417..5330f31 100644
--- a/content/docs/latest/sql/z-order-introduction.html
+++ b/content/docs/latest/sql/z-order-introduction.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>3. Z-order introduction — Kyuubi 1.5.0-incubating documentation</title>
+ <title>3. Z-order introduction — Kyuubi 1.5.1-incubating documentation</title>
@@ -352,18 +352,18 @@
<p>If you want to optimize the history data of a table, the <code class="docutils literal notranslate"><span class="pre">OPTIMIZE</span> <span class="pre">...</span></code> syntax is good to go. Due to Spark SQL doesn’t support read and overwrite same datasource table, the syntax can only support to optimize Hive table.</p>
<div class="section" id="syntax">
<h4>Syntax<a class="headerlink" href="#syntax" title="Permalink to this headline">¶</a></h4>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="n">OPTIMIZE</span> <span class="k">table_name</span> <span class="p">[</span><span class="k">WHERE</span> <span class="n">predicate</span><span class="p">]</span> <span class="n">ZORDER</span> <span class="k">BY</span> <span class="n">col_name1</span> <span class="p">[,</span> <span class="p">...]</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="n">OPTIMIZE</span><span class="w"> </span><span class="k">table_name</span><span class="w"> </span><span class="p">[</span><span class="k">WHERE</span><span class="w"> </span><span class="n">predicate</span><span class="p">]</span><span class="w"> </span><span class="n">ZORDER</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">col_name1</span><span clas [...]
</pre></div>
</div>
<p>Note that, the <code class="docutils literal notranslate"><span class="pre">predicate</span></code> only supports partition spec.</p>
</div>
<div class="section" id="examples">
<h4>Examples<a class="headerlink" href="#examples" title="Permalink to this headline">¶</a></h4>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="n">OPTIMIZE</span> <span class="n">t1</span> <span class="n">ZORDER</span> <span class="k">BY</span> <span class="n">c3</span><span class="p">;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="n">OPTIMIZE</span><span class="w"> </span><span class="n">t1</span><span class="w"> </span><span class="n">ZORDER</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">c3</span><span class="p">;</span><span class="w"></span>
-<span class="n">OPTIMIZE</span> <span class="n">t1</span> <span class="n">ZORDER</span> <span class="k">BY</span> <span class="n">c1</span><span class="p">,</span><span class="n">c2</span><span class="p">;</span>
+<span class="n">OPTIMIZE</span><span class="w"> </span><span class="n">t1</span><span class="w"> </span><span class="n">ZORDER</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">c1</span><span class="p">,</span><span class="n">c2</span><span class="p">;</span><span class="w"></span>
-<span class="n">OPTIMIZE</span> <span class="n">t1</span> <span class="k">WHERE</span> <span class="k">day</span> <span class="o">=</span> <span class="s1">'2021-12-01'</span> <span class="n">ZORDER</span> <span class="k">BY</span> <span class="n">c1</span><span class="p">,</span><span class="n">c2</span><span class="p">;</span>
+<span class="n">OPTIMIZE</span><span class="w"> </span><span class="n">t1</span><span class="w"> </span><span class="k">WHERE</span><span class="w"> </span><span class="k">day</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'2021-12-01'</span><span class="w"> </span><span class="n">ZORDER</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">c1</span><span class="p">,</span><span class="n">c2< [...]
</pre></div>
</div>
</div>
@@ -371,7 +371,7 @@
<div class="section" id="optimize-incremental-data">
<h3><span class="section-number">3.2.2. </span>Optimize incremental data<a class="headerlink" href="#optimize-incremental-data" title="Permalink to this headline">¶</a></h3>
<p>Kyuubi supports optimize a table automatically for incremental data. e.g., time partitioned table. The only things you need to do is adding Kyuubi properties into the target table properties:</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">ALTER</span> <span class="k">TABLE</span> <span class="n">t1</span> <span class="k">SET</span> <span class="n">TBLPROPERTIES</span><span class="p">(</span><span class="s1">'kyuubi.zorder.enabled'</span><span class="o">=</span><span class="s1">'true'</span><span class="p">,</span><span class="s1">'kyuubi.zorder.cols'</span><span class="o">=</span><span class="s1">'c1 [...]
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">ALTER</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">t1</span><span class="w"> </span><span class="k">SET</span><span class="w"> </span><span class="n">TBLPROPERTIES</span><span class="p">(</span><span class="s1">'kyuubi.zorder.enabled'</span><span class="o">=</span><span class="s1">'true'</span><span class="p">,</span><span cl [...]
</pre></div>
</div>
<ul class="simple">
@@ -379,11 +379,11 @@
<li><p>the key <code class="docutils literal notranslate"><span class="pre">kyuubi.zorder.cols</span></code> decide which columns are used to optimize by z-order.</p></li>
</ul>
<p>Kyuubi will detect the properties and optimize SQL using z-order during SQL compilation, so you can enjoy z-order with all writing table command like:</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">INSERT</span> <span class="k">INTO</span> <span class="k">TABLE</span> <span class="n">t1</span> <span class="n">PARTITION</span><span class="p">()</span> <span class="p">...;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">INSERT</span><span class="w"> </span><span class="k">INTO</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">t1</span><span class="w"> </span><span class="n">PARTITION</span><span class="p">()</span><span class="w"> </span><span class="p">...;</span><span class="w"></span>
-<span class="k">INSERT</span> <span class="n">OVERWRITE</span> <span class="k">TABLE</span> <span class="n">t1</span> <span class="n">PARTITION</span><span class="p">()</span> <span class="p">...;</span>
+<span class="k">INSERT</span><span class="w"> </span><span class="n">OVERWRITE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">t1</span><span class="w"> </span><span class="n">PARTITION</span><span class="p">()</span><span class="w"> </span><span class="p">...;</span><span class="w"></span>
-<span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">t1</span> <span class="k">AS</span> <span class="k">SELECT</span> <span class="p">...;</span>
+<span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">t1</span><span class="w"> </span><span class="k">AS</span><span class="w"> </span><span class="k">SELECT</span><span class="w"> </span><span class="p">...;</span><span class="w"></span>
</pre></div>
</div>
</div>
diff --git a/content/docs/latest/tools/index.html b/content/docs/latest/tools/index.html
index 2e1c6fc..e1222d4 100644
--- a/content/docs/latest/tools/index.html
+++ b/content/docs/latest/tools/index.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Tools — Kyuubi 1.5.0-incubating documentation</title>
+ <title>Tools — Kyuubi 1.5.1-incubating documentation</title>
diff --git a/content/docs/latest/tools/spark_block_cleaner.html b/content/docs/latest/tools/spark_block_cleaner.html
index 46b0dd4..7cb35d0 100644
--- a/content/docs/latest/tools/spark_block_cleaner.html
+++ b/content/docs/latest/tools/spark_block_cleaner.html
@@ -7,7 +7,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>1. Kubernetes Tools Spark Block Cleaner — Kyuubi 1.5.0-incubating documentation</title>
+ <title>1. Kubernetes Tools Spark Block Cleaner — Kyuubi 1.5.1-incubating documentation</title>
@@ -252,53 +252,53 @@
<p>You need to modify the <code class="docutils literal notranslate"><span class="pre">${KYUUBI_HOME}/tools/spark-block-cleaner/kubernetes/spark-block-cleaner.yml</span></code> to fit your current environment.</p>
<p>In Kyuubi tools, we recommend using <code class="docutils literal notranslate"><span class="pre">DaemonSet</span></code> to start, and we offer default yaml file in daemonSet way.</p>
<p>Base file structure:</p>
-<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">apiVersion</span>
-<span class="l l-Scalar l-Scalar-Plain">kind</span>
-<span class="l l-Scalar l-Scalar-Plain">metadata</span>
- <span class="l l-Scalar l-Scalar-Plain">name</span>
- <span class="l l-Scalar l-Scalar-Plain">namespace</span>
-<span class="l l-Scalar l-Scalar-Plain">spec</span>
- <span class="l l-Scalar l-Scalar-Plain">select</span>
- <span class="l l-Scalar l-Scalar-Plain">template</span>
- <span class="l l-Scalar l-Scalar-Plain">metadata</span>
- <span class="l l-Scalar l-Scalar-Plain">spce</span>
- <span class="l l-Scalar l-Scalar-Plain">containers</span>
- <span class="l l-Scalar l-Scalar-Plain">- image</span>
- <span class="l l-Scalar l-Scalar-Plain">- volumeMounts</span>
- <span class="l l-Scalar l-Scalar-Plain">- env</span>
- <span class="l l-Scalar l-Scalar-Plain">volumes</span>
+<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">apiVersion</span><span class="w"></span>
+<span class="l l-Scalar l-Scalar-Plain">kind</span><span class="w"></span>
+<span class="l l-Scalar l-Scalar-Plain">metadata</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">name</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">namespace</span><span class="w"></span>
+<span class="l l-Scalar l-Scalar-Plain">spec</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">select</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">template</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">metadata</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">spce</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">containers</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">- image</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">- volumeMounts</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">- env</span><span class="w"></span>
+<span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">volumes</span><span class="w"></span>
</pre></div>
</div>
<p>You can use affect the performance of Spark Block Cleaner through configure parameters in containers env part of <code class="docutils literal notranslate"><span class="pre">spark-block-cleaner.yml</span></code>.</p>
-<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">env</span><span class="p">:</span>
- <span class="p p-Indicator">-</span> <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">CACHE_DIRS</span>
- <span class="nt">value</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">/data/data1,/data/data2</span>
- <span class="p p-Indicator">-</span> <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">FILE_EXPIRED_TIME</span>
- <span class="nt">value</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">604800</span>
- <span class="p p-Indicator">-</span> <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">DEEP_CLEAN_FILE_EXPIRED_TIME</span>
- <span class="nt">value</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">432000</span>
- <span class="p p-Indicator">-</span> <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">FREE_SPACE_THRESHOLD</span>
- <span class="nt">value</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">60</span>
- <span class="p p-Indicator">-</span> <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">SCHEDULE_INTERVAL</span>
- <span class="nt">value</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">3600</span>
+<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">env</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">CACHE_DIRS</span><span class="w"></span>
+<span class="w"> </span><span class="nt">value</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/data/data1,/data/data2</span><span class="w"></span>
+<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">FILE_EXPIRED_TIME</span><span class="w"></span>
+<span class="w"> </span><span class="nt">value</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">604800</span><span class="w"></span>
+<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">DEEP_CLEAN_FILE_EXPIRED_TIME</span><span class="w"></span>
+<span class="w"> </span><span class="nt">value</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">432000</span><span class="w"></span>
+<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">FREE_SPACE_THRESHOLD</span><span class="w"></span>
+<span class="w"> </span><span class="nt">value</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">60</span><span class="w"></span>
+<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">SCHEDULE_INTERVAL</span><span class="w"></span>
+<span class="w"> </span><span class="nt">value</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">3600</span><span class="w"></span>
</pre></div>
</div>
<p>The most important thing, configure volumeMounts and volumes corresponding to Spark local-dirs.</p>
<p>For example, Spark use /spark/shuffle1 as local-dir, you can configure like:</p>
-<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">volumes</span><span class="p">:</span>
- <span class="p p-Indicator">-</span> <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">block-files-dir-1</span>
- <span class="nt">hostPath</span><span class="p">:</span>
- <span class="nt">path</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">/spark/shuffle1</span>
+<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">volumes</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">block-files-dir-1</span><span class="w"></span>
+<span class="w"> </span><span class="nt">hostPath</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/spark/shuffle1</span><span class="w"></span>
</pre></div>
</div>
-<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">volumeMounts</span><span class="p">:</span>
- <span class="p p-Indicator">-</span> <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">block-files-dir-1</span>
- <span class="nt">mountPath</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">/data/data1</span>
+<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">volumeMounts</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">block-files-dir-1</span><span class="w"></span>
+<span class="w"> </span><span class="nt">mountPath</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/data/data1</span><span class="w"></span>
</pre></div>
</div>
-<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">env</span><span class="p">:</span>
- <span class="p p-Indicator">-</span> <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">CACHE_DIRS</span>
- <span class="nt">value</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">/data/data1</span>
+<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">env</span><span class="p">:</span><span class="w"></span>
+<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">CACHE_DIRS</span><span class="w"></span>
+<span class="w"> </span><span class="nt">value</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/data/data1</span><span class="w"></span>
</pre></div>
</div>
</div>