You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2019/09/11 15:14:11 UTC
[carbondata-site] branch asf-site updated: Added 1.5.4 & 1.6.0
version information
This is an automated email from the ASF dual-hosted git repository.
raghunandan pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/carbondata-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 8ac54a5 Added 1.5.4 & 1.6.0 version information
8ac54a5 is described below
commit 8ac54a51ac1b6d94ea3019929d4dcfa51ecf8573
Author: Raghunandan S <ca...@gmail.com>
AuthorDate: Mon Sep 2 22:41:43 2019 +0530
Added 1.5.4 & 1.6.0 version information
---
content/CSDK-guide.html | 9 +
content/WEB-INF/classes/application.conf | 2 +
content/alluxio-guide.html | 9 +
content/bloomfilter-datamap-guide.html | 9 +
content/carbon-as-spark-datasource-guide.html | 31 +-
content/configuration-parameters.html | 34 ++
content/datamap-developer-guide.html | 9 +
content/datamap-management.html | 16 +-
content/ddl-of-carbondata.html | 68 ++-
content/dml-of-carbondata.html | 25 +-
content/documentation.html | 9 +
content/faq.html | 9 +
content/file-structure-of-carbondata.html | 9 +
content/hive-guide.html | 49 +-
.../how-to-contribute-to-apache-carbondata.html | 9 +
content/index-server.html | 572 +++++++++++++++++++++
content/index.html | 22 +-
content/introduction.html | 9 +
content/language-manual.html | 9 +
content/lucene-datamap-guide.html | 9 +
.../mv-datamap-guide.html | 284 ++++++----
content/performance-tuning.html | 9 +
content/preaggregate-datamap-guide.html | 28 +-
content/presto-guide.html | 9 +
content/quick-start-guide.html | 9 +
content/release-guide.html | 9 +
content/s3-guide.html | 9 +
content/sdk-guide.html | 85 ++-
content/security.html | 6 +
content/segment-management-on-carbondata.html | 9 +
content/streaming-guide.html | 15 +-
content/supported-data-types-in-carbondata.html | 10 +
content/timeseries-datamap-guide.html | 9 +
content/usecases.html | 9 +
content/videogallery.html | 6 +
src/main/resources/application.conf | 2 +
src/main/scala/html/header.html | 9 +
src/main/scala/scripts/index-server | 4 +
src/main/scala/scripts/mv-datamap-guide | 11 +
src/main/webapp/CSDK-guide.html | 9 +
src/main/webapp/alluxio-guide.html | 9 +
src/main/webapp/bloomfilter-datamap-guide.html | 9 +
.../webapp/carbon-as-spark-datasource-guide.html | 31 +-
src/main/webapp/configuration-parameters.html | 34 ++
src/main/webapp/datamap-developer-guide.html | 9 +
src/main/webapp/datamap-management.html | 16 +-
src/main/webapp/ddl-of-carbondata.html | 68 ++-
src/main/webapp/dml-of-carbondata.html | 25 +-
src/main/webapp/documentation.html | 9 +
src/main/webapp/faq.html | 9 +
src/main/webapp/file-structure-of-carbondata.html | 9 +
src/main/webapp/hive-guide.html | 49 +-
.../how-to-contribute-to-apache-carbondata.html | 9 +
src/main/webapp/index-server.html | 572 +++++++++++++++++++++
src/main/webapp/index.html | 6 +
src/main/webapp/introduction.html | 9 +
src/main/webapp/language-manual.html | 9 +
src/main/webapp/lucene-datamap-guide.html | 9 +
...ne-datamap-guide.html => mv-datamap-guide.html} | 284 ++++++----
src/main/webapp/performance-tuning.html | 9 +
src/main/webapp/preaggregate-datamap-guide.html | 28 +-
src/main/webapp/presto-guide.html | 9 +
src/main/webapp/quick-start-guide.html | 9 +
src/main/webapp/release-guide.html | 9 +
src/main/webapp/s3-guide.html | 9 +
src/main/webapp/sdk-guide.html | 85 ++-
src/main/webapp/security.html | 6 +
.../webapp/segment-management-on-carbondata.html | 9 +
src/main/webapp/streaming-guide.html | 15 +-
.../webapp/supported-data-types-in-carbondata.html | 10 +
src/main/webapp/timeseries-datamap-guide.html | 9 +
src/main/webapp/usecases.html | 9 +
src/main/webapp/videogallery.html | 6 +
.../markdown/carbon-as-spark-datasource-guide.md | 20 +-
src/site/markdown/configuration-parameters.md | 5 +
src/site/markdown/datamap-management.md | 8 +-
src/site/markdown/ddl-of-carbondata.md | 54 +-
src/site/markdown/dml-of-carbondata.md | 8 +
src/site/markdown/hive-guide.md | 39 +-
src/site/markdown/index-server.md | 227 ++++++++
src/site/markdown/mv-datamap-guide.md | 208 ++++++++
src/site/markdown/preaggregate-datamap-guide.md | 3 +
src/site/markdown/sdk-guide.md | 84 ++-
.../markdown/supported-data-types-in-carbondata.md | 2 +-
84 files changed, 3174 insertions(+), 340 deletions(-)
diff --git a/content/CSDK-guide.html b/content/CSDK-guide.html
index a7f2b76..1c252a5 100644
--- a/content/CSDK-guide.html
+++ b/content/CSDK-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/WEB-INF/classes/application.conf b/content/WEB-INF/classes/application.conf
index 430fa47..eb1c327 100644
--- a/content/WEB-INF/classes/application.conf
+++ b/content/WEB-INF/classes/application.conf
@@ -21,12 +21,14 @@ fileList=["configuration-parameters",
"carbon-as-spark-datasource-guide",
"alluxio-guide",
"hive-guide",
+ "index-server",
"presto-guide"
]
dataMapFileList=[
"bloomfilter-datamap-guide",
"lucene-datamap-guide",
"preaggregate-datamap-guide",
+ "mv-datamap-guide",
"timeseries-datamap-guide",
"datamap-management"]
diff --git a/content/alluxio-guide.html b/content/alluxio-guide.html
index b6bcdd7..b148273 100644
--- a/content/alluxio-guide.html
+++ b/content/alluxio-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/bloomfilter-datamap-guide.html b/content/bloomfilter-datamap-guide.html
index cc7ee14..51682cc 100644
--- a/content/bloomfilter-datamap-guide.html
+++ b/content/bloomfilter-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/carbon-as-spark-datasource-guide.html b/content/carbon-as-spark-datasource-guide.html
index 9c8224a..d19bfe9 100644
--- a/content/carbon-as-spark-datasource-guide.html
+++ b/content/carbon-as-spark-datasource-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -221,7 +230,7 @@
<h1>
<a id="carbondata-as-sparks-datasource" class="anchor" href="#carbondata-as-sparks-datasource" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData as Spark's Datasource</h1>
<p>The CarbonData fileformat is now integrated as Spark datasource for read and write operation without using CarbonSession. This is useful for users who wants to use carbondata as spark's data source.</p>
-<p><strong>Note:</strong> You can only apply the functions/features supported by spark datasource APIs, functionalities supported would be similar to Parquet. The carbon session features are not supported.</p>
+<p><strong>Note:</strong> You can only apply the functions/features supported by spark datasource APIs, functionalities supported would be similar to Parquet. The carbon session features are not supported. The result is displayed as byte array format when select query on binary column in spark-sql.</p>
<h1>
<a id="create-table-with-ddl" class="anchor" href="#create-table-with-ddl" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create Table with DDL</h1>
<p>Now you can create Carbon table using Spark's datasource DDL syntax.</p>
@@ -250,7 +259,7 @@
<tr>
<td>table_blocksize</td>
<td>1024</td>
-<td>Size of blocks to write onto hdfs. For more details, see <a href="./ddl-of-carbondata.html#table-block-size-configuration">Table Block Size Configuration</a>.</td>
+<td>Size of blocks to write onto hdfs. For more details, see <a href="./ddl-of-carbondata.html#table-block-size-configuration">Table Block Size Configuration</a>.</td>
</tr>
<tr>
<td>table_blocklet_size</td>
@@ -258,35 +267,41 @@
<td>Size of blocklet to write.</td>
</tr>
<tr>
+<td>table_page_size_inmb</td>
+<td>0</td>
+<td>Size of each page in carbon table, if page size crosses this value before 32000 rows, page will be cut to that many rows. Helps in keep page size to fit cache size</td>
+</tr>
+<tr>
<td>local_dictionary_threshold</td>
<td>10000</td>
-<td>Cardinality upto which the local dictionary can be generated. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
+<td>Cardinality upto which the local dictionary can be generated. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
</tr>
<tr>
<td>local_dictionary_enable</td>
<td>false</td>
-<td>Enable local dictionary generation. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
+<td>Enable local dictionary generation. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
</tr>
<tr>
<td>sort_columns</td>
<td>all dimensions are sorted</td>
-<td>Columns to include in sort and its order of sort. For more details, see <a href="./ddl-of-carbondata.html#sort-columns-configuration">Sort Columns Configuration</a>.</td>
+<td>Columns to include in sort and its order of sort. For more details, see <a href="./ddl-of-carbondata.html#sort-columns-configuration">Sort Columns Configuration</a>.</td>
</tr>
<tr>
<td>sort_scope</td>
<td>local_sort</td>
-<td>Sort scope of the load.Options include no sort, local sort, batch sort, and global sort. For more details, see <a href="./ddl-of-carbondata.html#sort-scope-configuration">Sort Scope Configuration</a>.</td>
+<td>Sort scope of the load.Options include no sort, local sort, batch sort, and global sort. For more details, see <a href="./ddl-of-carbondata.html#sort-scope-configuration">Sort Scope Configuration</a>.</td>
</tr>
<tr>
<td>long_string_columns</td>
<td>null</td>
-<td>Comma separated string/char/varchar columns which are more than 32k length. For more details, see <a href="./ddl-of-carbondata.html#string-longer-than-32000-characters">String longer than 32000 characters</a>.</td>
+<td>Comma separated string/char/varchar columns which are more than 32k length. For more details, see <a href="./ddl-of-carbondata.html#string-longer-than-32000-characters">String longer than 32000 characters</a>.</td>
</tr>
</tbody>
</table>
+<p><strong>NOTE:</strong> please set long_string_columns for varchar column.</p>
<h2>
<a id="example" class="anchor" href="#example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example</h2>
-<pre><code> CREATE TABLE CARBON_TABLE (NAME STRING) USING CARBON OPTIONS('table_block_size'='256')
+<pre><code> CREATE TABLE CARBON_TABLE (NAME STRING) USING CARBON OPTIONS('table_block_size'='256')
</code></pre>
<h1>
<a id="using-dataframe" class="anchor" href="#using-dataframe" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Using DataFrame</h1>
diff --git a/content/configuration-parameters.html b/content/configuration-parameters.html
index ca4b9e9..776d64a 100644
--- a/content/configuration-parameters.html
+++ b/content/configuration-parameters.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -326,6 +335,11 @@
<td>5</td>
<td>Specifies the interval between the retries to obtain the lock for any operation other than load. <strong>NOTE:</strong> Refer to <em><strong>carbon.lock.retries</strong></em> for understanding why CarbonData uses locks for operations.</td>
</tr>
+<tr>
+<td>carbon.fs.custom.file.provider</td>
+<td>None</td>
+<td>To support FileTypeInterface for configuring custom CarbonFile implementation to work with custom FileSystem.</td>
+</tr>
</tbody>
</table>
<h2>
@@ -540,6 +554,16 @@
<td>200</td>
<td>CarbonData will write the min max values for string/varchar types column using the byte count specified by this configuration. Max value is 1000 bytes(500 characters) and Min value is 10 bytes(5 characters). <strong>NOTE:</strong> This property is useful for reducing the store size thereby improving the query performance but can lead to query degradation if value is not configured properly.</td>
</tr>
+<tr>
+<td>carbon.merge.index.failure.throw.exception</td>
+<td>true</td>
+<td>It is used to configure whether or not merge index failure should result in data load failure also.</td>
+</tr>
+<tr>
+<td>carbon.binary.decoder</td>
+<td>None</td>
+<td>Support configurable decode for loading. Two decoders supported: base64 and hex</td>
+</tr>
</tbody>
</table>
<h2>
@@ -624,6 +648,11 @@
<td>true</td>
<td>Each CarbonData file has a companion CarbonIndex file which maintains the metadata about the data. These CarbonIndex files are read and loaded into driver and is used subsequently for pruning of data during queries. These CarbonIndex files are very small in size(few KB) and are many. Reading many small files from HDFS is not efficient and leads to slow IO performance. Hence these CarbonIndex files belonging to a segment can be combined into a single file and read once there by incre [...]
</tr>
+<tr>
+<td>carbon.enable.range.compaction</td>
+<td>true</td>
+<td>To configure Ranges-based Compaction to be used or not for RANGE_COLUMN. If true after compaction also the data would be present in ranges.</td>
+</tr>
</tbody>
</table>
<h2>
@@ -743,6 +772,11 @@
<td>false</td>
<td>When enabled complete row filters will be handled by carbon in case of vector. If it is disabled then only page level pruning will be done by carbon and row level filtering will be done by spark for vector. And also there are scan optimizations in carbon to avoid multiple data copies when this parameter is set to false. There is no change in flow for non-vector based queries.</td>
</tr>
+<tr>
+<td>carbon.query.prefetch.enable</td>
+<td>true</td>
+<td>By default this property is true, so prefetch is used in query to read next blocklet asynchronously in other thread while processing current blocklet in main thread. This can help to reduce CPU idle time. Setting this property false will disable this prefetch feature in query.</td>
+</tr>
</tbody>
</table>
<h2>
diff --git a/content/datamap-developer-guide.html b/content/datamap-developer-guide.html
index e5ec135..09da4cc 100644
--- a/content/datamap-developer-guide.html
+++ b/content/datamap-developer-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/datamap-management.html b/content/datamap-management.html
index 172dc00..a4d7461 100644
--- a/content/datamap-management.html
+++ b/content/datamap-management.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -274,7 +283,7 @@ AS
<td>mv</td>
<td>multi-table pre-aggregate table</td>
<td>No DMPROPERTY is required</td>
-<td>Manual</td>
+<td>Manual/Automatic</td>
</tr>
<tr>
<td>lucene</td>
@@ -297,8 +306,6 @@ AS
<li>Automatic Refresh: Create datamap without <code>WITH DEFERRED REBUILD</code> in the statement, which is by default.</li>
<li>Manual Refresh: Create datamap with <code>WITH DEFERRED REBUILD</code> in the statement</li>
</ol>
-<p><strong>CAUTION:</strong>
-If user create MV datamap without specifying <code>WITH DEFERRED REBUILD</code>, carbondata will give a warning and treat the datamap as deferred rebuild.</p>
<h3>
<a id="automatic-refresh" class="anchor" href="#automatic-refresh" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Automatic Refresh</h3>
<p>When user creates a datamap on the main table without using <code>WITH DEFERRED REBUILD</code> syntax, the datamap will be managed by system automatically.
@@ -361,6 +368,9 @@ Scan Table: default.datamap1_table
<li>DataMapProviderName like mv, preaggreagte, timeseries, etc</li>
<li>Associated Table</li>
<li>DataMap Properties</li>
+<li>DataMap status (ENABLED/DISABLED)</li>
+<li>Sync Status - which displays Last segment Id of main table synced with datamap table and its load
+end time (Applicable only for mv datamap)</li>
</ul>
<h3>
<a id="compaction-on-datamap" class="anchor" href="#compaction-on-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compaction on DataMap</h3>
diff --git a/content/ddl-of-carbondata.html b/content/ddl-of-carbondata.html
index 8dea391..5720f10 100644
--- a/content/ddl-of-carbondata.html
+++ b/content/ddl-of-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -356,6 +365,10 @@ STORED AS carbondata
<td>Size of blocklet to write in the file</td>
</tr>
<tr>
+<td><a href="#table-page-size-configuration">TABLE_PAGE_SIZE_INMB</a></td>
+<td>Size of page in MB; if page size crosses this value before 32000 rows, page will be cut to this many rows and remaining rows are processed in the subsequent pages. This helps in keeping page size to fit in cpu cache size</td>
+</tr>
+<tr>
<td><a href="#table-compaction-configuration">MAJOR_COMPACTION_SIZE</a></td>
<td>Size upto which the segments can be combined into one</td>
</tr>
@@ -440,7 +453,7 @@ Suggested use cases : do dictionary encoding for low cardinality columns, it mig
</code></pre>
<p><strong>NOTE</strong>:</p>
<ul>
-<li>Dictionary Include/Exclude for complex child columns is not supported.</li>
+<li>Dictionary Include/Exclude for complex child columns is not supported. Dictionary Include doesn't support binary data type.</li>
<li>Dictionary is global. Except global dictionary, there are local dictionary and non-dictionary in CarbonData.</li>
</ul>
</li>
@@ -480,6 +493,7 @@ Suggested use cases : do dictionary encoding for low cardinality columns, it mig
<li>BOOLEAN</li>
<li>FLOAT</li>
<li>BYTE</li>
+<li>Binary</li>
</ul>
</li>
<li>
@@ -498,8 +512,8 @@ Suggested use cases : do dictionary encoding for low cardinality columns, it mig
<tbody>
<tr>
<td>carbon.local.dictionary.enable</td>
-<td>false</td>
-<td>By default, Local Dictionary will be disabled for the carbondata table.</td>
+<td>true</td>
+<td>By default, Local Dictionary will be enabled for the carbondata table.</td>
</tr>
<tr>
<td>carbon.local.dictionary.decoder.fallback</td>
@@ -589,14 +603,14 @@ Suggested use cases : For high cardinality columns, you can disable the inverted
<a id="sort-columns-configuration" class="anchor" href="#sort-columns-configuration" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Sort Columns Configuration</h5>
<p>This property is for users to specify which columns belong to the MDK(Multi-Dimensions-Key) index.</p>
<ul>
-<li>If users don't specify "SORT_COLUMN" property, by default no columns are sorted</li>
+<li>If users don't specify "SORT_COLUMNS" property, by default no columns are sorted</li>
<li>If this property is specified but with empty argument, then the table will be loaded without sort.</li>
<li>This supports only string, date, timestamp, short, int, long, byte and boolean data types.
Suggested use cases : Only build MDK index for required columns,it might help to improve the data loading performance.</li>
</ul>
<pre><code>TBLPROPERTIES ('SORT_COLUMNS'='column1, column3')
</code></pre>
-<p><strong>NOTE</strong>: Sort_Columns for Complex datatype columns is not supported.</p>
+<p><strong>NOTE</strong>: Sort_Columns for Complex datatype columns and binary data type is not supported.</p>
</li>
<li>
<h5>
@@ -647,6 +661,21 @@ Blocklet is the minimum IO read unit, in case of point queries reduce blocklet s
</li>
<li>
<h5>
+<a id="table-page-size-configuration" class="anchor" href="#table-page-size-configuration" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Table page Size Configuration</h5>
+<p>This property is for setting page size in the carbondata file
+and supports a range of 1 MB to 1755 MB.
+If page size crosses this value before 32000 rows, page will be cut to that many rows.
+Helps in keeping page size to fit cpu cache size.</p>
+<p>This property can be configured if the table has string, varchar, binary or complex datatype columns.
+Because for these columns 32000 rows in one page may exceed 1755 MB and snappy compression will fail in that scenario.
+Also if page size is huge, page cannot be fit in CPU cache.
+So, configuring smaller values of this property (say 1 MB) can result in better use of CPU cache for pages.</p>
+<p>Example usage:</p>
+<pre><code>TBLPROPERTIES ('TABLE_PAGE_SIZE_INMB'='5')
+</code></pre>
+</li>
+<li>
+<h5>
<a id="table-compaction-configuration" class="anchor" href="#table-compaction-configuration" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Table Compaction Configuration</h5>
<p>These properties are table level compaction configurations, if not specified, system level configurations in carbon.properties will be used.
Following are 5 configurations:</p>
@@ -674,7 +703,7 @@ Following are 5 configurations:</p>
<li>
<h5>
<a id="caching-minmax-value-for-required-columns" class="anchor" href="#caching-minmax-value-for-required-columns" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Caching Min/Max Value for Required Columns</h5>
-<p>By default, CarbonData caches min and max values of all the columns in schema. As the load increases, the memory required to hold the min and max values increases considerably. This feature enables you to configure min and max values only for the required columns, resulting in optimized memory usage.</p>
+<p>By default, CarbonData caches min and max values of all the columns in schema. As the load increases, the memory required to hold the min and max values increases considerably. This feature enables you to configure min and max values only for the required columns, resulting in optimized memory usage. This feature doesn't support binary data type.</p>
<p>Following are the valid values for COLUMN_META_CACHE:</p>
<ul>
<li>If you want no column min/max values to be cached in the driver.</li>
@@ -813,7 +842,8 @@ Notice that once you enable this feature, for load balance, carbondata will igno
<h5>
<a id="range-column" class="anchor" href="#range-column" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Range Column</h5>
<p>This property is used to specify a column to partition the input data by range.
-Only one column can be configured. During data loading, you can use "global_sort_partitions" or "scale_factor" to avoid generating small files.</p>
+Only one column can be configured. During data loading, you can use "global_sort_partitions" or "scale_factor" to avoid generating small files.
+This feature doesn't support binary data type.</p>
<pre><code>TBLPROPERTIES('RANGE_COLUMN'='col1')
</code></pre>
</li>
@@ -1048,6 +1078,28 @@ This can be achieved by using the alter table set command.</p>
<pre><code>ALTER TABLE tablename UNSET TBLPROPERTIES('SORT_SCOPE')
</code></pre>
</li>
+<li>
+<h5>
+<a id="sort-columns" class="anchor" href="#sort-columns" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>SORT COLUMNS</h5>
+<p>Example to SET SORT COLUMNS:</p>
+<pre><code>ALTER TABLE tablename SET TBLPROPERTIES('SORT_COLUMNS'='column1')
+</code></pre>
+<p>After this operation, the new loading will use the new SORT_COLUMNS. The user can adjust
+the SORT_COLUMNS according to the query, but it will not impact the old data directly. So
+it will not impact the query performance of the old data segments which are not sorted by
+new SORT_COLUMNS.</p>
+<p>UNSET is not supported, but it can set SORT_COLUMNS to empty string instead of using UNSET.
+NOTE: When SORT_SCOPE is not NO_SORT, then setting SORT_COLUMNS to empty string is not valid.</p>
+<pre><code>ALTER TABLE tablename SET TBLPROPERTIES('SORT_COLUMNS'='')
+</code></pre>
+<p><strong>NOTE:</strong></p>
+<ul>
+<li>The future version will enhance "custom" compaction to sort the old segment one by one.</li>
+<li>The streaming table is not supported for SORT_COLUMNS modification.</li>
+<li>If the inverted index columns are removed from the new SORT_COLUMNS, they will not
+create the inverted index. But the old configuration of INVERTED_INDEX will be kept.</li>
+</ul>
+</li>
</ul>
</li>
</ul>
@@ -1124,7 +1176,7 @@ TBLPROPERTIES ('DICTIONARY_INCLUDE'='productNumber')
PARTITIONED BY (productCategory STRING, productBatch STRING)
STORED AS carbondata
</code></pre>
-<p><strong>NOTE:</strong> Hive partition is not supported on complex datatype columns.</p>
+<p><strong>NOTE:</strong> Hive partition is not supported on complex data type columns.</p>
<h4>
<a id="show-partitions" class="anchor" href="#show-partitions" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Show Partitions</h4>
<p>This command gets the Hive partition information of the table</p>
diff --git a/content/dml-of-carbondata.html b/content/dml-of-carbondata.html
index eaf4476..05f848c 100644
--- a/content/dml-of-carbondata.html
+++ b/content/dml-of-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -343,6 +352,10 @@ OPTIONS(property_name=property_value, ...)
<td><a href="#scale_factor">SCALE_FACTOR</a></td>
<td>Control the partition size for RANGE_COLUMN feature</td>
</tr>
+<tr>
+<td>[CARBON_OPTIONS_BINARY_DECODER]</td>
+<td>Support configurable decode for loading from csv</td>
+</tr>
</tbody>
</table>
<ul>
@@ -550,7 +563,9 @@ OPTIONS('BAD_RECORDS_LOGGER_ENABLE'='true','BAD_RECORD_PATH'='hdfs://hacluster/t
<h5>
<a id="global_sort_partitions" class="anchor" href="#global_sort_partitions" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>GLOBAL_SORT_PARTITIONS:</h5>
<p>If the SORT_SCOPE is defined as GLOBAL_SORT, then user can specify the number of partitions to use while shuffling data for sort using GLOBAL_SORT_PARTITIONS. If it is not configured, or configured less than 1, then it uses the number of map task as reduce task. It is recommended that each reduce task deal with 512MB-1GB data.
-For RANGE_COLUMN, GLOBAL_SORT_PARTITIONS is used to specify the number of range partitions also.</p>
+For RANGE_COLUMN, GLOBAL_SORT_PARTITIONS is used to specify the number of range partitions also.
+GLOBAL_SORT_PARTITIONS should be specified optimally during RANGE_COLUMN LOAD because if a higher number is configured then the load time may be less but it will result in creation of more files which would degrade the query and compaction performance.
+Conversely, if less partitions are configured then the load performance may degrade due to less use of parallelism but the query and compaction will become faster. Hence the user may choose optimal number depending on the use case.</p>
</li>
</ul>
<pre><code>OPTIONS('GLOBAL_SORT_PARTITIONS'='2')
@@ -578,6 +593,14 @@ For RANGE_COLUMN, GLOBAL_SORT_PARTITIONS is used to specify the number of range
<li>If both GLOBAL_SORT_PARTITIONS and SCALE_FACTOR are used at the same time, only GLOBAL_SORT_PARTITIONS is valid.</li>
<li>The compaction on RANGE_COLUMN will use LOCAL_SORT by default.</li>
</ul>
+<ul>
+<li>
+<h5>
+<a id="carbon_enable_range_compaction" class="anchor" href="#carbon_enable_range_compaction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CARBON_ENABLE_RANGE_COMPACTION</h5>
+</li>
+</ul>
+<p>To configure Ranges-based Compaction to be used or not for RANGE_COLUMN.
+The default value is 'true'.</p>
<h3>
diff --git a/content/documentation.html b/content/documentation.html
index b502f10..3ca010d 100644
--- a/content/documentation.html
+++ b/content/documentation.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/faq.html b/content/faq.html
index 49b793c..579b867 100644
--- a/content/faq.html
+++ b/content/faq.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/file-structure-of-carbondata.html b/content/file-structure-of-carbondata.html
index 10518af..e4975d4 100644
--- a/content/file-structure-of-carbondata.html
+++ b/content/file-structure-of-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/hive-guide.html b/content/hive-guide.html
index 1c11ed7..296b959 100644
--- a/content/hive-guide.html
+++ b/content/hive-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -222,9 +231,6 @@
<a id="quick-start" class="anchor" href="#quick-start" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Quick Start</h1>
<p>This tutorial provides a quick introduction to using current integration/hive module.</p>
<h2>
-<a id="build-in-120-hive-integration-only-support-spark21-and-hadoop272" class="anchor" href="#build-in-120-hive-integration-only-support-spark21-and-hadoop272" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Build (In 1.2.0, hive integration only support spark2.1 and hadoop2.7.2)</h2>
-<p>mvn -DskipTests -Pspark-2.1 -Phadoop-2.7.2 clean package</p>
-<h2>
<a id="prepare-carbondata-in-spark" class="anchor" href="#prepare-carbondata-in-spark" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Prepare CarbonData in Spark</h2>
<ul>
<li>
@@ -285,18 +291,41 @@ export HIVE_AUX_JARS_PATH=hive/auxlibs/
export HADOOP_OPTS="-Dorg.xerial.snappy.lib.path=/Library/Java/Extensions -Dorg.xerial.snappy.lib.name=libsnappyjava.jnilib -Dorg.xerial.snappy.tempdir=/Users/apple/DEMO/tmp"
</code></pre>
<h3>
-<a id="start-hive-client" class="anchor" href="#start-hive-client" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Start hive client</h3>
-<pre><code>$HIVE_HOME/bin/hive
+<a id="carbon-jars-to-be-placed" class="anchor" href="#carbon-jars-to-be-placed" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Carbon Jars to be placed</h3>
+<pre><code>hive/lib/ (for hive server)
+yarn/lib/ (for MapReduce)
+
+Carbon Jars to be copied to the above paths.
</code></pre>
<h3>
-<a id="query-data-from-hive-table" class="anchor" href="#query-data-from-hive-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Query data from hive table</h3>
+<a id="start-hive-beeline-to-query" class="anchor" href="#start-hive-beeline-to-query" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Start hive beeline to query</h3>
+<pre><code>$HIVE_HOME/bin/beeline
+</code></pre>
+<h3>
+<a id="query-data-from-hive" class="anchor" href="#query-data-from-hive" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Query data from hive</h3>
+<ul>
+<li>This is to read the carbon table through Hive. It is the integration of the carbon with Hive.</li>
+</ul>
<pre><code>set hive.mapred.supports.subdirectories=true;
-set mapreduce.input.fileinputformat.input.dir.recursive=true;
+set mapreduce.dir.recursive=true;
+These properties helps to recursively traverse through the directories to read the carbon folder structure.
+</code></pre>
+<h3>
+<a id="example" class="anchor" href="#example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example</h3>
+<pre><code> - In case if the carbon table is not set with the SERDE and the INPUTFORMAT/OUTPUTFORMAT, user can create a new hive managed table like below with the required details for the hive to read.
+create table hive_carbon_1(id int, name string, scale decimal, country string, salary double) ROW FORMAT SERDE 'org.apache.carbondata.hive.CarbonHiveSerDe' WITH SERDEPROPERTIES ('mapreduce.input.carboninputformat.databaseName'='default', 'mapreduce.input.carboninputformat.tableName'='HIVE_CARBON_EXAMPLE') STORED AS INPUTFORMAT 'org.apache.carbondata.hive.MapredCarbonInputFormat' OUTPUTFORMAT 'org.apache.carbondata.hive.MapredCarbonOutputFormat' LOCATION 'location_to_the_carbon_table';
-select * from hive_carbon;
-select count(*) from hive_carbon;
-select * from hive_carbon order by id;
+ - Query the table
+select * from hive_carbon_1;
+select count(*) from hive_carbon_1;
+select * from hive_carbon_1 order by id;
</code></pre>
+<h3>
+<a id="note" class="anchor" href="#note" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Note</h3>
+<ul>
+<li>Partition table support is not handled</li>
+<li>Map data type is not supported</li>
+</ul>
<script>
// Show selected style on nav item
$(function() { $('.b-nav__quickstart').addClass('selected'); });
diff --git a/content/how-to-contribute-to-apache-carbondata.html b/content/how-to-contribute-to-apache-carbondata.html
index 92ef5e4..28139df 100644
--- a/content/how-to-contribute-to-apache-carbondata.html
+++ b/content/how-to-contribute-to-apache-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/index-server.html b/content/index-server.html
new file mode 100644
index 0000000..f7bea90
--- /dev/null
+++ b/content/index-server.html
@@ -0,0 +1,572 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'>
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <title>CarbonData</title>
+ <style>
+
+ </style>
+ <!-- Bootstrap -->
+
+ <link rel="stylesheet" href="css/bootstrap.min.css">
+ <link href="css/style.css" rel="stylesheet">
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="js/jquery.min.js"></script>
+ <script src="js/bootstrap.min.js"></script>
+ <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js"></script>
+
+
+</head>
+<body>
+<header>
+ <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper">
+ <div class="container">
+ <div class="navbar-header">
+ <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse"
+ class="navbar-toggle collapsed" type="button">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="index.html" class="logo">
+ <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/>
+ </a>
+ </div>
+ <div class="navbar-collapse collapse cd_navcontnt" id="navbar">
+ <ul class="nav navbar-nav navbar-right navlist-custom">
+ <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a>
+ </li>
+ <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false"> Download <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
+ target="_blank">Apache CarbonData 1.5.3</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.2/"
+ target="_blank">Apache CarbonData 1.5.2</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.1/"
+ target="_blank">Apache CarbonData 1.5.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
+ target="_blank">Release Archive</a></li>
+ </ul>
+ </li>
+ <li><a href="documentation.html" class="active">Documentation</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false">Community <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/how-to-contribute-to-apache-carbondata.md"
+ target="_blank">Contributing to CarbonData</a></li>
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"
+ target="_blank">Release Guide</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"
+ target="_blank">Project PMC and Committers</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"
+ target="_blank">CarbonData Meetups</a></li>
+ <li><a href="security.html">Apache CarbonData Security</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/CARBONDATA" target="_blank">Apache
+ Jira</a></li>
+ <li><a href="videogallery.html">CarbonData Videos </a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="apache_link hidden-xs dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="hidden-lg hidden-md hidden-sm dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li>
+ <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a>
+
+ </li>
+
+ </ul>
+ </div><!--/.nav-collapse -->
+ <div id="search-box">
+ <form method="get" action="http://www.google.com/search" target="_blank">
+ <div class="search-block">
+ <table border="0" cellpadding="0" width="100%">
+ <tr>
+ <td style="width:80%">
+ <input type="text" name="q" size=" 5" maxlength="255" value=""
+ class="search-input" placeholder="Search...." required/>
+ </td>
+ <td style="width:20%">
+ <input type="submit" value="Search"/></td>
+ </tr>
+ <tr>
+ <td align="left" style="font-size:75%" colspan="2">
+ <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/>
+ <span style=" position: relative; top: -3px;"> Only search for CarbonData</span>
+ </td>
+ </tr>
+ </table>
+ </div>
+ </form>
+ </div>
+ </div>
+ </nav>
+</header> <!-- end Header part -->
+
+<div class="fixed-padding"></div> <!-- top padding with fixde header -->
+
+<section><!-- Dashboard nav -->
+ <div class="container-fluid q">
+ <div class="col-sm-12 col-md-12 maindashboard">
+ <div class="verticalnavbar">
+ <nav class="b-sticky-nav">
+ <div class="nav-scroller">
+ <div class="nav__inner">
+ <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a>
+ <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a>
+ <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a>
+ <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a>
+ <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a>
+ <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a>
+ <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a>
+ <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a>
+ <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a>
+ </div>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a>
+ <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a>
+ <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
+ <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
+ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
+ </div>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
+ <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
+ <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
+ <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
+ <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
+ <a class="b-nav__security nav__item" href="./security.html">Security</a>
+ <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a>
+ </div>
+ </div>
+ <div class="navindicator">
+ <div class="b-nav__intro navindicator__item"></div>
+ <div class="b-nav__quickstart navindicator__item"></div>
+ <div class="b-nav__uses navindicator__item"></div>
+ <div class="b-nav__docs navindicator__item"></div>
+ <div class="b-nav__datamap navindicator__item"></div>
+ <div class="b-nav__api navindicator__item"></div>
+ <div class="b-nav__perf navindicator__item"></div>
+ <div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
+ <div class="b-nav__faq navindicator__item"></div>
+ <div class="b-nav__contri navindicator__item"></div>
+ <div class="b-nav__security navindicator__item"></div>
+ </div>
+ </nav>
+ </div>
+ <div class="mdcontent">
+ <section>
+ <div style="padding:10px 15px;">
+ <div id="viewpage" name="viewpage">
+ <div class="row">
+ <div class="col-sm-12 col-md-12">
+ <div>
+<h1>
+<a id="distributed-index-server" class="anchor" href="#distributed-index-server" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Distributed Index Server</h1>
+<h2>
+<a id="background" class="anchor" href="#background" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Background</h2>
+<p>Carbon currently prunes and caches all block/blocklet datamap index information into the driver for
+normal table, for Bloom/Index datamaps the JDBC driver will launch a job to prune and cache the
+datamaps in executors.</p>
+<p>This causes the driver to become a bottleneck in the following ways:</p>
+<ol>
+<li>If the cache size becomes huge(70-80% of the driver memory) then there can be excessive GC in
+the driver which can slow down the query and the driver may even go OutOfMemory.</li>
+<li>LRU has to evict a lot of elements from the cache to accommodate the new objects which would
+in turn slow down the queries.</li>
+<li>For bloom there is no guarantee that the next query goes to the same executor to reuse the cache
+and hence cache could be duplicated in multiple executors.</li>
+<li>Multiple JDBC drivers need to maintain their own copy of the cache.</li>
+</ol>
+<p>Distributed Index Cache Server aims to solve the above mentioned problems.</p>
+<h2>
+<a id="distribution" class="anchor" href="#distribution" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Distribution</h2>
+<p>When enabled, any query on a carbon table will be routed to the index server service in form of
+a request. The request will consist of the table name, segments, filter expression and other
+information used for pruning.</p>
+<p>In IndexServer service a pruning RDD is fired which will take care of the pruning for that
+request. This RDD will be creating tasks based on the number of segments that are applicable for
+pruning. It can happen that the user has specified segments to access for that table, so only the
+specified segments would be applicable for pruning. Refer: <a href="https://github.com/apache/carbondata/blob/6e50c1c6fc1d6e82a4faf6dc6e0824299786ccc0/docs/segment-management-on-carbondata.html#query-data-with-specified-segments">query-data-with-specified-segments</a>.
+IndexServer driver would have 2 important tasks, distributing the segments equally among the
+available executors and keeping track of the executor where the segment is cached.</p>
+<p>To achieve this 2 separate mappings would be maintained as follows.</p>
+<ol>
+<li>
+<p>segment to executor location:
+This mapping will be maintained for each table and will enable the index server to track the
+cache location for each segment.</p>
+</li>
+<li>
+<p>Cache size held by each executor:
+This mapping will be used to distribute the segments equally(on the basis of size) among the
+executors.</p>
+</li>
+</ol>
+<p>Once a request is received each segment would be iterated over and
+checked against tableToExecutorMapping to find if a executor is already
+assigned. If a mapping already exists then it means that most
+probably(if not evicted by LRU) the segment is already cached in that
+executor and the task for that segment has to be fired on this executor.</p>
+<p>If mapping is not found then first check executorToCacheMapping against
+the available executor list to find if any unassigned executor is
+present and use that executor for the current segment. If all the
+executors are assigned with some segment then find the least loaded
+executor on the basis of size.</p>
+<p>Initially the segment index size would be used to distribute the
+segments fairly among the executor because the actual cache size would
+be known to the driver only when the segments are cached and appropriate
+information is returned to the driver.</p>
+<p><strong>NOTE:</strong> In case of legacy segment(version: 1.1) the index size is not available
+therefore all the legacy segments would be processed in a round robin
+fashion.</p>
+<p>After the job is completed the tasks would return the cache size held by
+each executor which would be updated to the executorToCacheMapping and
+the pruned blocklets which would be further used for result fetching.</p>
+<p><strong>Note:</strong> Multiple JDBC drivers can connect to the index server to use the cache.</p>
+<h2>
+<a id="reallocation-of-executor" class="anchor" href="#reallocation-of-executor" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Reallocation of executor</h2>
+<p>In case executor(s) become dead/unavailable then the segments that were
+earlier being handled by those would be reassigned to some other
+executor using the distribution logic.</p>
+<p><strong>Note:</strong> Cache loading would be done again in the new executor for the
+current query.</p>
+<h2>
+<a id="metacache-ddl" class="anchor" href="#metacache-ddl" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>MetaCache DDL</h2>
+<p>The show metacache DDL has a new column called cache location will indicate whether the cache is
+from executor or driver. To drop cache the user has to enable/disable the index server using the
+dynamic configuration to clear the cache of the desired location.</p>
+<p>Refer: <a href="https://github.com/apache/carbondata/blob/master/docs/ddl-of-carbondata.html#cache">MetaCacheDDL</a></p>
+<h2>
+<a id="fallback" class="anchor" href="#fallback" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Fallback</h2>
+<p>In case of any failure the index server would fallback to embedded mode
+which means that the JDBCServer would take care of distributed pruning.
+A similar job would be fired by the JDBCServer which would take care of
+pruning using its own executors. If for any reason the embedded mode
+also fails to prune the datamaps then the job would be passed on to
+driver.</p>
+<p><strong>NOTE:</strong> In case of embedded mode a job would be fired after pruning to clear the
+cache as data cached in JDBCServer executors would be of no use.</p>
+<h2>
+<a id="writing-splits-to-a-file" class="anchor" href="#writing-splits-to-a-file" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Writing splits to a file</h2>
+<p>If the response is too huge then it is better to write the splits to a file so that the driver can
+read this file and create the splits. This can be controlled using the property 'carbon.index.server
+.inmemory.serialization.threshold.inKB'. By default, the minimum value for this property is 0,
+meaning that no matter how small the splits are they would be written to the file. Maximum is
+102400KB which will mean if the size of the splits for a executor cross this value then they would
+be written to file.</p>
+<p>The user can set the location for these file by using 'carbon.indexserver.temp.path'. By default
+table path would be used to write the files.</p>
+<h2>
+<a id="security" class="anchor" href="#security" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Security</h2>
+<p>The security for the index server is controlled through 'spark.carbon.indexserver.keytab' and 'spark
+.carbon.indexserver.principal'. These allow the RPC framework to login using the principal. It is
+recommended that the principal should be a super user, and the user should be exclusive for index
+server so that it does not grant access to any other service. Internally the operations would be
+executed as a Privileged Action using the login user.</p>
+<p>The Index Server is a long running service therefore the 'spark.yarn.keytab' and 'spark.yarn
+.principal' should be configured.</p>
+<h2>
+<a id="configurations" class="anchor" href="#configurations" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Configurations</h2>
+<h5>
+<a id="carbonpropertiesjdbcserver" class="anchor" href="#carbonpropertiesjdbcserver" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>carbon.properties(JDBCServer)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">carbon.enable.index.server</td>
+<td align="center">false</td>
+<td align="center">Enable the use of index server for pruning for the whole application.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.ip</td>
+<td align="center">NA</td>
+<td align="center">Specify the IP/HOST on which the server is started. Better to specify the private IP.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.port</td>
+<td align="center">NA</td>
+<td align="center">The port on which the index server is started.</td>
+</tr>
+<tr>
+<td align="center">carbon.disable.index.server.fallback</td>
+<td align="center">false</td>
+<td align="center">Whether to enable/disable fallback for index server. Should be used for testing purposes only. Refer: <a href="#Fallback">Fallback</a>
+</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.max.jobname.length</td>
+<td align="center">NA</td>
+<td align="center">The max length of the job to show in the index server service UI. For bigger queries this may impact performance as the whole string would be sent from JDBCServer to IndexServer.</td>
+</tr>
+</tbody>
+</table>
+<h5>
+<a id="carbonpropertiesindexserver" class="anchor" href="#carbonpropertiesindexserver" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>carbon.properties(IndexServer)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">carbon.index.server.ip</td>
+<td align="center">NA</td>
+<td align="center">Specify the IP/HOST on which the server would be started. Better to specify the private IP.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.port</td>
+<td align="center">NA</td>
+<td align="center">The port on which the index server has to be started.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.max.worker.threads</td>
+<td align="center">500</td>
+<td align="center">Number of RPC handlers to open for accepting the requests from JDBC driver. Max accepted value is Integer.Max. Refer: <a href="https://github.com/apache/hive/blob/master/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java#L3441">Hive configuration</a>
+</td>
+</tr>
+<tr>
+<td align="center">carbon.max.executor.lru.cache.size</td>
+<td align="center">NA</td>
+<td align="center">Maximum memory <strong>(in MB)</strong> upto which the executor process can cache the data (DataMaps and reverse dictionary values). Only integer values greater than 0 are accepted. <strong>NOTE:</strong> Mandatory for the user to set.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.max.jobname.length</td>
+<td align="center">NA</td>
+<td align="center">The max length of the job to show in the index server application UI. For bigger queries this may impact performance as the whole string would be sent from JDBCServer to IndexServer.</td>
+</tr>
+<tr>
+<td align="center">carbon.max.executor.threads.for.block.pruning</td>
+<td align="center">4</td>
+<td align="center">max executor threads used for block pruning.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.inmemory.serialization.threshold.inKB</td>
+<td align="center">300</td>
+<td align="center">Max in memory serialization size after reaching threshold data will be written to file. Min value that the user can set is 0KB and max is 102400KB.</td>
+</tr>
+<tr>
+<td align="center">carbon.indexserver.temp.path</td>
+<td align="center">tablePath</td>
+<td align="center">The folder to write the split files if in memory datamap size for network transfers crossed the 'carbon.index.server.inmemory.serialization.threshold.inKB' limit.</td>
+</tr>
+</tbody>
+</table>
+<h5>
+<a id="spark-defaultsconfonly-for-secure-mode" class="anchor" href="#spark-defaultsconfonly-for-secure-mode" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>spark-defaults.conf(only for secure mode)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">spark.carbon.indexserver.principal</td>
+<td align="center">NA</td>
+<td align="center">Used for authentication, whether a valid service is trying to connect to the server or not. Set in both IndexServer and JDBCServer.</td>
+</tr>
+<tr>
+<td align="center">spark.carbon.indexserver.keytab</td>
+<td align="center">NA</td>
+<td align="center">Specify the path to the keytab file through which authentication would happen. Set in both IndexServer and JDBCServer.</td>
+</tr>
+<tr>
+<td align="center">spark.dynamicAllocation.enabled</td>
+<td align="center">true</td>
+<td align="center">Set to false, so that spark does not kill the executor, If executors are killed, cache would be lost. Applicable only for Index Server.</td>
+</tr>
+<tr>
+<td align="center">spark.yarn.principal</td>
+<td align="center">NA</td>
+<td align="center">Should be set to the same user used for JDBCServer. Required only for IndexServer.</td>
+</tr>
+<tr>
+<td align="center">spark.yarn.keytab</td>
+<td align="center">NA</td>
+<td align="center">Should be set to the same as JDBCServer.</td>
+</tr>
+</tbody>
+</table>
+<h5>
+<a id="spark-defaultsconfnon-secure-mode" class="anchor" href="#spark-defaultsconfnon-secure-mode" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>spark-defaults.conf(non-secure mode)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">spark.dynamicAllocation.enabled</td>
+<td align="center">true</td>
+<td align="center">Set to false, so that spark does not kill the executor, If executors are killed, cache would be lost. Applicable only for Index Server.</td>
+</tr>
+</tbody>
+</table>
+<p><strong>NOTE:</strong> Its better to create a new user for indexserver principal,
+that will authenticate the user to access the index server and no other service.</p>
+<h5>
+<a id="core-sitexml" class="anchor" href="#core-sitexml" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>core-site.xml</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">ipc.client.rpc-timeout.ms</td>
+<td align="center">NA</td>
+<td align="center">Set the above property to some appropriate value based on your estimated query time. The best option is to set this to the same value as spark.network.timeout.</td>
+</tr>
+</tbody>
+</table>
+<h5>
+<a id="dynamic-propertiesset-command" class="anchor" href="#dynamic-propertiesset-command" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>dynamic-properties(set command)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">carbon.enable.index.server</td>
+<td align="center">false</td>
+<td align="center">Enable the use of index server for pruning for the current session.</td>
+</tr>
+<tr>
+<td align="center">carbon.enable.index.server.dbName.tableName</td>
+<td align="center">false</td>
+<td align="center">Enable the use of index server for the specified table in the current session.</td>
+</tr>
+</tbody>
+</table>
+<h2>
+<a id="starting-the-server" class="anchor" href="#starting-the-server" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Starting the Server</h2>
+<pre><code>./bin/spark-submit --master [yarn/local] --[o ptional parameters] --class org.apache.carbondata.indexserver.IndexServer [path to carbondata-spark2-<version>.jar]
+</code></pre>
+<p>Or</p>
+<pre><code>./sbin/start-indexserver.sh --master yarn --num-executors 2 /<absolute path>/carbondata-spark2-1.6.0.0100.jar
+</code></pre>
+<h2>
+<a id="faq" class="anchor" href="#faq" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>FAQ</h2>
+<p>Q. <strong>Index Server is throwing Large response size exception.</strong></p>
+<p>A. The exception would show the size of response it is trying to send over the
+network. Use ipc.maximum.response.length to a value bigger than the
+response size.</p>
+<p>Q. <strong>Index server is throwing Kerberos principal not set exception</strong></p>
+<p>A. Set spark.carbon.indexserver.principal to the correct principal in both IndexServer and
+JDBCServer configurations.</p>
+<p>Q. <strong>Unable to connect to index server</strong></p>
+<p>A. Check whether the carbon.properties configurations are set in JDBCServer as well as the index
+server.</p>
+<p>Q. <strong>IndexServer is throwing FileNotFoundException for index files.</strong></p>
+<p>A. Check whether the Index server and JDBCServer are connected to the
+same namenode or not. And the store should be shared by both</p>
+<p>Q. <strong>OutOfMemoryException in DirectMemoryBuffer</strong></p>
+<p>A. Increase -XX:MaxDirectMemorySize in driver.extraJavaOptions to
+accommodate the large response in driver.</p>
+<script>
+// Show selected style on nav item
+$(function() { $('.b-nav__indexserver').addClass('selected'); });
+</script></div>
+</div>
+</div>
+</div>
+<div class="doc-footer">
+ <a href="#top" class="scroll-top">Top</a>
+</div>
+</div>
+</section>
+</div>
+</div>
+</div>
+</section><!-- End systemblock part -->
+<script src="js/custom.js"></script>
+</body>
+</html>
\ No newline at end of file
diff --git a/content/index.html b/content/index.html
index e1f33cb..8a3aab0 100644
--- a/content/index.html
+++ b/content/index.html
@@ -54,6 +54,12 @@
class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -313,6 +319,20 @@
</h4>
<div class="linkblock">
<div class="block-row">
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a>
+ <span class="release-date">Sept 2019</span>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.6.0+Release"
+ class="whatsnew" target="_blank">what's new</a>
+ </div>
+ <div class="block-row">
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a>
+ <span class="release-date">July 2019</span>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.5.4+Release"
+ class="whatsnew" target="_blank">what's new</a>
+ </div>
+ <div class="block-row">
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a>
<span class="release-date">Apr 2019</span>
@@ -478,7 +498,7 @@
to do is:</p>
<ol class="orderlist">
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
target="_blank">Download</a>the latest release.
</li>
diff --git a/content/introduction.html b/content/introduction.html
index 8af7902..79731fa 100644
--- a/content/introduction.html
+++ b/content/introduction.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/language-manual.html b/content/language-manual.html
index 0b7bca3..6e3cfbb 100644
--- a/content/language-manual.html
+++ b/content/language-manual.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/lucene-datamap-guide.html b/content/lucene-datamap-guide.html
index 4f3290c..db90362 100644
--- a/content/lucene-datamap-guide.html
+++ b/content/lucene-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/lucene-datamap-guide.html b/content/mv-datamap-guide.html
similarity index 58%
copy from src/main/webapp/lucene-datamap-guide.html
copy to content/mv-datamap-guide.html
index 4f3290c..a6a3ee2 100644
--- a/src/main/webapp/lucene-datamap-guide.html
+++ b/content/mv-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -219,153 +228,202 @@
<div class="col-sm-12 col-md-12">
<div>
<h1>
-<a id="carbondata-lucene-datamap-alpha-feature" class="anchor" href="#carbondata-lucene-datamap-alpha-feature" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Lucene DataMap (Alpha Feature)</h1>
+<a id="carbondata-mv-datamap" class="anchor" href="#carbondata-mv-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData MV DataMap</h1>
<ul>
-<li><a href="#datamap-management">DataMap Management</a></li>
-<li><a href="#lucene-datamap-introduction">Lucene Datamap</a></li>
+<li><a href="#quick-example">Quick Example</a></li>
+<li><a href="#mv-datamap-introduction">MV DataMap</a></li>
<li><a href="#loading-data">Loading Data</a></li>
<li><a href="#querying-data">Querying Data</a></li>
-<li><a href="#data-management-with-lucene-datamap">Data Management</a></li>
+<li><a href="#compacting-mv-tables">Compaction</a></li>
+<li><a href="#data-management-with-mv-tables">Data Management</a></li>
</ul>
-<h4>
-<a id="datamap-management" class="anchor" href="#datamap-management" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>DataMap Management</h4>
-<p>Lucene DataMap can be created using following DDL</p>
-<pre><code>CREATE DATAMAP [IF NOT EXISTS] datamap_name
-ON TABLE main_table
-USING 'lucene'
-DMPROPERTIES ('index_columns'='city, name', ...)
-</code></pre>
-<p>DataMap can be dropped using following DDL:</p>
-<pre><code>DROP DATAMAP [IF EXISTS] datamap_name
-ON TABLE main_table
-</code></pre>
-<p>To show all DataMaps created, use:</p>
-<pre><code>SHOW DATAMAP
-ON TABLE main_table
+<h2>
+<a id="quick-example" class="anchor" href="#quick-example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Quick example</h2>
+<p>Start spark-sql in terminal and run the following queries,</p>
+<pre><code>CREATE TABLE maintable(a int, b string, c int) stored by 'carbondata';
+insert into maintable select 1, 'ab', 2;
+CREATE DATAMAP datamap_1 on table maintable as SELECT a, sum(b) from maintable group by a;
+SELECT a, sum(b) from maintable group by a;
+// NOTE: run explain query and check if query hits the datamap table from the plan
+EXPLAIN SELECT a, sum(b) from maintable group by a;
</code></pre>
-<p>It will show all DataMaps created on main table.</p>
<h2>
-<a id="lucene-datamap-introduction" class="anchor" href="#lucene-datamap-introduction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Lucene DataMap Introduction</h2>
-<p>Lucene is a high performance, full featured text search engine. Lucene is integrated to carbon as
-an index datamap and managed along with main tables by CarbonData. User can create lucene datamap
-to improve query performance on string columns which has content of more length. So, user can
-search tokenized word or pattern of it using lucene query on text content.</p>
-<p>For instance, main table called <strong>datamap_test</strong> which is defined as:</p>
-<pre><code>CREATE TABLE datamap_test (
- name string,
- age int,
- city string,
- country string)
+<a id="mv-datamap-introduction" class="anchor" href="#mv-datamap-introduction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>MV DataMap Introduction</h2>
+<p>MV tables are created as DataMaps and managed as tables internally by CarbonData. User can create
+limitless MV datamaps on a table to improve query performance provided the storage requirements
+and loading time is acceptable.</p>
+<p>MV datamap can be a lazy or a non-lazy datamap. Once MV datamaps are created, CarbonData's
+CarbonAnalyzer helps to select the most efficient MV datamap based on the user query and rewrite
+the SQL to select the data from MV datamap instead of main table. Since the data size of MV
+datamap is smaller and data is pre-processed, user queries are much faster.</p>
+<p>For instance, main table called <strong>sales</strong> which is defined as</p>
+<pre><code>CREATE TABLE sales (
+ order_time timestamp,
+ user_id string,
+ sex string,
+ country string,
+ quantity int,
+ price bigint)
STORED AS carbondata
</code></pre>
-<p>User can create Lucene datamap using the Create DataMap DDL:</p>
-<pre><code>CREATE DATAMAP dm
-ON TABLE datamap_test
-USING 'lucene'
-DMPROPERTIES ('INDEX_COLUMNS' = 'name, country',)
+<p>User can create MV tables using the Create DataMap DDL</p>
+<pre><code>CREATE DATAMAP agg_sales
+ON TABLE sales
+USING "MV"
+AS
+ SELECT country, sex, sum(quantity), avg(price)
+ FROM sales
+ GROUP BY country, sex
</code></pre>
-<p><strong>DMProperties</strong></p>
+<p><strong>NOTE</strong>:</p>
+<ul>
+<li>
+<p>Group by/Filter columns has to be provided in projection list while creating mv datamap</p>
+</li>
+<li>
+<p>If only single parent table is involved in mv datamap creation, then TableProperties of Parent table
+(if not present in a aggregate function like sum(col)) listed below will be
+inherited to datamap table</p>
<ol>
-<li>INDEX_COLUMNS: The list of string columns on which lucene creates indexes.</li>
-<li>FLUSH_CACHE: size of the cache to maintain in Lucene writer, if specified then it tries to
-aggregate the unique data till the cache limit and flush to Lucene. It is best suitable for low
-cardinality dimensions.</li>
-<li>SPLIT_BLOCKLET: when made as true then store the data in blocklet wise in lucene , it means new
-folder will be created for each blocklet, thus, it eliminates storing blockletid in lucene and
-also it makes lucene small chunks of data.</li>
+<li>SORT_COLUMNS</li>
+<li>SORT_SCOPE</li>
+<li>TABLE_BLOCKSIZE</li>
+<li>FLAT_FOLDER</li>
+<li>LONG_STRING_COLUMNS</li>
+<li>LOCAL_DICTIONARY_ENABLE</li>
+<li>LOCAL_DICTIONARY_THRESHOLD</li>
+<li>LOCAL_DICTIONARY_EXCLUDE</li>
+<li>DICTIONARY_INCLUDE</li>
+<li>DICTIONARY_EXCLUDE</li>
+<li>INVERTED_INDEX</li>
+<li>NO_INVERTED_INDEX</li>
+<li>COLUMN_COMPRESSOR</li>
</ol>
-<h2>
-<a id="loading-data" class="anchor" href="#loading-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data</h2>
-<p>When loading data to main table, lucene index files will be generated for all the
-index_columns(String Columns) given in DMProperties which contains information about the data
-location of index_columns. These index files will be written inside a folder named with datamap name
-inside each segment folders.</p>
-<p>A system level configuration carbon.lucene.compression.mode can be added for best compression of
-lucene index files. The default value is speed, where the index writing speed will be more. If the
-value is compression, the index file size will be compressed.</p>
-<h2>
-<a id="querying-data" class="anchor" href="#querying-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Querying data</h2>
-<p>As a technique for query acceleration, Lucene indexes cannot be queried directly.
-Queries are to be made on main table. when a query with TEXT_MATCH('name:c10') or
-TEXT_MATCH_WITH_LIMIT('name:n10',10)[the second parameter represents the number of result to be
-returned, if user does not specify this value, all results will be returned without any limit] is
-fired, two jobs are fired. The first job writes the temporary files in folder created at table level
-which contains lucene's seach results and these files will be read in second job to give faster
-results. These temporary files will be cleared once the query finishes.</p>
-<p>User can verify whether a query can leverage Lucene datamap or not by executing <code>EXPLAIN</code>
-command, which will show the transformed logical plan, and thus user can check whether TEXT_MATCH()
-filter is applied on query or not.</p>
-<p><strong>Note:</strong></p>
-<ol>
+</li>
<li>
-<p>The filter columns in TEXT_MATCH or TEXT_MATCH_WITH_LIMIT must be always in lower case and
-filter condition like 'AND','OR' must be in upper case.</p>
-<p>Ex:</p>
-<pre><code>select * from datamap_test where TEXT_MATCH('name:*10 AND name:*n*')
-</code></pre>
+<p>All columns of main table at once cannot participate in mv datamap table creation</p>
</li>
<li>
-<p>Query supports only one TEXT_MATCH udf for filter condition and not multiple udfs.</p>
-<p>The following query is supported:</p>
-<pre><code>select * from datamap_test where TEXT_MATCH('name:*10 AND name:*n*')
-</code></pre>
-<p>The following query is not supported:</p>
-<pre><code>select * from datamap_test where TEXT_MATCH('name:*10) AND TEXT_MATCH(name:*n*')
-</code></pre>
+<p>TableProperties can be provided in DMProperties excluding LOCAL_DICTIONARY_INCLUDE,
+LOCAL_DICTIONARY_EXCLUDE, DICTIONARY_INCLUDE, DICTIONARY_EXCLUDE, INVERTED_INDEX,
+NO_INVERTED_INDEX, SORT_COLUMNS, LONG_STRING_COLUMNS, RANGE_COLUMN & COLUMN_META_CACHE</p>
</li>
-</ol>
-<p>Below like queries can be converted to text_match queries as following:</p>
-<pre><code>select * from datamap_test where name='n10'
-
-select * from datamap_test where name like 'n1%'
-
-select * from datamap_test where name like '%10'
+<li>
+<p>TableProperty given in DMProperties will be considered for mv creation, eventhough if same
+property is inherited from parent table, which allows user to provide different tableproperties
+for child table</p>
+</li>
+<li>
+<p>MV creation with limit or union all ctas queries is unsupported</p>
+</li>
+</ul>
+<h4>
+<a id="how-mv-tables-are-selected" class="anchor" href="#how-mv-tables-are-selected" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>How MV tables are selected</h4>
+<p>When a user query is submitted, during query planning phase, CarbonData will collect modular plan
+candidates and process the the ModularPlan based on registered summary data sets. Then,
+mv datamap table for this query will be selected among the candidates.</p>
+<p>For the main table <strong>sales</strong> and mv table <strong>agg_sales</strong> created above, following queries</p>
+<pre><code>SELECT country, sex, sum(quantity), avg(price) from sales GROUP BY country, sex
-select * from datamap_test where name like '%n%'
+SELECT sex, sum(quantity) from sales GROUP BY sex
-select * from datamap_test where name like '%10' and name not like '%n%'
+SELECT avg(price), country from sales GROUP BY country
</code></pre>
-<p>Lucene TEXT_MATCH Queries:</p>
-<pre><code>select * from datamap_test where TEXT_MATCH('name:n10')
-
-select * from datamap_test where TEXT_MATCH('name:n1*')
-
-select * from datamap_test where TEXT_MATCH('name:*10')
+<p>will be transformed by CarbonData's query planner to query against mv table
+<strong>agg_sales</strong> instead of the main table <strong>sales</strong></p>
+<p>However, for following queries</p>
+<pre><code>SELECT user_id, country, sex, sum(quantity), avg(price) from sales GROUP BY user_id, country, sex
-select * from datamap_test where TEXT_MATCH('name:*n*')
+SELECT sex, avg(quantity) from sales GROUP BY sex
-select * from datamap_test where TEXT_MATCH('name:*10 -name:*n*')
+SELECT country, max(price) from sales GROUP BY country
</code></pre>
-<p><strong>Note:</strong> For lucene queries and syntax, refer to <a href="http://www.lucenetutorial.com/lucene-query-syntax.html" target=_blank rel="nofollow">lucene-syntax</a></p>
+<p>will query against main table <strong>sales</strong> only, because it does not satisfy mv table
+selection logic.</p>
<h2>
-<a id="data-management-with-lucene-datamap" class="anchor" href="#data-management-with-lucene-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Data Management with lucene datamap</h2>
-<p>Once there is lucene datamap is created on the main table, following command on the main
-table
-is not supported:</p>
+<a id="loading-data" class="anchor" href="#loading-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data</h2>
+<h3>
+<a id="loading-data-to-non-lazy-mv-datamap" class="anchor" href="#loading-data-to-non-lazy-mv-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data to Non-Lazy MV Datamap</h3>
+<p>In case of WITHOUT DEFERRED REBUILD, for existing table with loaded data, data load to MV table will
+be triggered by the CREATE DATAMAP statement when user creates the MV table.
+For incremental loads to main table, data to datamap will be loaded once the corresponding main
+table load is completed.</p>
+<h3>
+<a id="loading-data-to-lazy-mv-datamap" class="anchor" href="#loading-data-to-lazy-mv-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data to Lazy MV Datamap</h3>
+<p>In case of WITH DEFERRED REBUILD, data load to MV table will be triggered by the <a href="./datamap-management.html#manual-refresh">Manual Refresh</a>
+command. MV datamap will be in DISABLED state in below scenarios,</p>
+<ul>
+<li>when mv datamap is created</li>
+<li>when data of main table and datamap are not in sync</li>
+</ul>
+<p>User should fire REBUILD DATAMAP command to sync all segments of main table with datamap table and
+which ENABLES the datamap for query</p>
+<h3>
+<a id="loading-data-to-multiple-mvs" class="anchor" href="#loading-data-to-multiple-mvs" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data to Multiple MV's</h3>
+<p>During load to main table, if anyone of the load to datamap table fails, then that corresponding
+datamap will be DISABLED and load to other datamaps mapped to main table will continue. User can
+fire REBUILD DATAMAP command to sync or else the subsequent table load will load the old failed
+loads along with current load and enable the disabled datamap.</p>
+<p><strong>NOTE</strong>:</p>
+<ul>
+<li>In case of InsertOverwrite/Update operation on parent table, all segments of datamap table will
+be MARKED_FOR_DELETE and reload to datamap table will happen by REBUILD DATAMAP, in case of Lazy
+mv datamap/ once InsertOverwrite/Update operation on parent table is finished, in case of
+Non-Lazy mv.</li>
+<li>In case of full scan query, Data Size and Index Size of main table and child table will not the
+same, as main table and child table has different column names.</li>
+</ul>
+<h2>
+<a id="querying-data" class="anchor" href="#querying-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Querying data</h2>
+<p>As a technique for query acceleration, MV tables cannot be queried directly.
+Queries are to be made on main table. While doing query planning, internally CarbonData will check
+associated mv datamap tables with the main table, and do query plan transformation accordingly.</p>
+<p>User can verify whether a query can leverage mv datamap table or not by executing <code>EXPLAIN</code>
+command, which will show the transformed logical plan, and thus user can check whether mv datamap
+table is selected.</p>
+<h2>
+<a id="compacting-mv-datamap" class="anchor" href="#compacting-mv-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compacting MV datamap</h2>
+<h3>
+<a id="compacting-mv-datamap-table-through-main-table-compaction" class="anchor" href="#compacting-mv-datamap-table-through-main-table-compaction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compacting MV datamap table through Main Table compaction</h3>
+<p>Running Compaction command (<code>ALTER TABLE COMPACT</code>)[COMPACTION TYPE-> MINOR/MAJOR] on main table will
+automatically compact the mv datamap tables created on the main table, once compaction on main table
+is done.</p>
+<h3>
+<a id="compacting-mv-datamap-table-through-ddl-command" class="anchor" href="#compacting-mv-datamap-table-through-ddl-command" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compacting MV datamap table through DDL command</h3>
+<p>Compaction on mv datamap can be triggered by running the following DDL command(supported only for mv).</p>
+<pre><code>ALTER DATAMAP datamap_name COMPACT 'COMPACTION_TYPE'
+</code></pre>
+<h2>
+<a id="data-management-with-mv-tables" class="anchor" href="#data-management-with-mv-tables" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Data Management with mv tables</h2>
+<p>In current implementation, data consistency needs to be maintained for both main table and mv datamap
+tables. Once there is mv datamap table created on the main table, following command on the main
+table is not supported:</p>
<ol>
-<li>Data management command: <code>UPDATE/DELETE</code>.</li>
+<li>Data management command: <code>DELETE SEGMENT</code>.</li>
<li>Schema management command: <code>ALTER TABLE DROP COLUMN</code>, <code>ALTER TABLE CHANGE DATATYPE</code>,
-<code>ALTER TABLE RENAME</code>.</li>
-</ol>
-<p><strong>Note</strong>: Adding a new column is supported, and for dropping columns and change datatype
-command, CarbonData will check whether it will impact the lucene datamap, if not, the operation
-is allowed, otherwise operation will be rejected by throwing exception.</p>
-<ol start="3">
-<li>Partition management command: <code>ALTER TABLE ADD/DROP PARTITION</code>.</li>
+<code>ALTER TABLE RENAME</code>, <code>ALTER COLUMN RENAME</code>. Note that adding a new column is supported, and for
+dropping columns and change datatype command, CarbonData will check whether it will impact the
+mv datamap table, if not, the operation is allowed, otherwise operation will be rejected by
+throwing exception.</li>
+<li>Partition management command: <code>ALTER TABLE ADD/DROP PARTITION</code>. Note that dropping a partition
+will be allowed only if partition is participating in all datamaps associated with main table.
+Drop Partition is not allowed, if any mv datamap is associated with more than one parent table.
+Drop Partition directly on datamap table is not allowed.</li>
+<li>Complex Datatype's for mv datamap is not supported.</li>
</ol>
<p>However, there is still way to support these operations on main table, in current CarbonData
release, user can do as following:</p>
<ol>
-<li>Remove the lucene datamap by <code>DROP DATAMAP</code> command.</li>
-<li>Carry out the data management operation on main table.</li>
-<li>Create the lucene datamap again by <code>CREATE DATAMAP</code> command.
+<li>Remove the mv datamap table by <code>DROP DATAMAP</code> command</li>
+<li>Carry out the data management operation on main table</li>
+<li>Create the mv datamap table again by <code>CREATE DATAMAP</code> command
Basically, user can manually trigger the operation by re-building the datamap.</li>
</ol>
<script>
$(function() {
// Show selected style on nav item
$('.b-nav__datamap').addClass('selected');
-
+
if (!$('.b-nav__datamap').parent().hasClass('nav__item__with__subs--expanded')) {
// Display datamap subnav items
$('.b-nav__datamap').parent().toggleClass('nav__item__with__subs--expanded');
diff --git a/content/performance-tuning.html b/content/performance-tuning.html
index 1fde679..2d93db9 100644
--- a/content/performance-tuning.html
+++ b/content/performance-tuning.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/preaggregate-datamap-guide.html b/content/preaggregate-datamap-guide.html
index b8c816d..ce95f90 100644
--- a/content/preaggregate-datamap-guide.html
+++ b/content/preaggregate-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -254,24 +263,24 @@
spark.sparkContext.setLogLevel(<span class="pl-s"><span class="pl-pds">"</span>ERROR<span class="pl-pds">"</span></span>)
<span class="pl-c"><span class="pl-c">//</span> drop table if exists previously</span>
- spark.sql(s<span class="pl-s"><span class="pl-pds">"</span>DROP TABLE IF EXISTS sales<span class="pl-pds">"</span></span>)
+ spark.sql(<span class="pl-k">s</span><span class="pl-s">"</span><span class="pl-s">DROP TABLE IF EXISTS sales</span><span class="pl-s">"</span>)
<span class="pl-c"><span class="pl-c">//</span> Create main table</span>
spark.sql(
- s<span class="pl-s"><span class="pl-pds">"""</span></span>
+ <span class="pl-k">s</span><span class="pl-s">"""</span>
<span class="pl-s"> | CREATE TABLE sales (</span>
<span class="pl-s"> | user_id string,</span>
<span class="pl-s"> | country string,</span>
<span class="pl-s"> | quantity int,</span>
<span class="pl-s"> | price bigint)</span>
<span class="pl-s"> | STORED AS carbondata</span>
-<span class="pl-s"> <span class="pl-pds">"""</span></span>.stripMargin)
+<span class="pl-s"> </span><span class="pl-s">"""</span>.stripMargin)
<span class="pl-c"><span class="pl-c">//</span> Create pre-aggregate table on the main table</span>
<span class="pl-c"><span class="pl-c">//</span> If main table already have data, following command </span>
<span class="pl-c"><span class="pl-c">//</span> will trigger one immediate load to the pre-aggregate table</span>
spark.sql(
- s<span class="pl-s"><span class="pl-pds">"""</span></span>
+ <span class="pl-k">s</span><span class="pl-s">"""</span>
<span class="pl-s"> | CREATE DATAMAP agg_sales</span>
<span class="pl-s"> | ON TABLE sales</span>
<span class="pl-s"> | USING "preaggregate"</span>
@@ -279,7 +288,7 @@
<span class="pl-s"> | SELECT country, sum(quantity), avg(price)</span>
<span class="pl-s"> | FROM sales</span>
<span class="pl-s"> | GROUP BY country</span>
-<span class="pl-s"> <span class="pl-pds">"""</span></span>.stripMargin)
+<span class="pl-s"> </span><span class="pl-s">"""</span>.stripMargin)
<span class="pl-k">import</span> <span class="pl-en">spark</span>.<span class="pl-en">implicits</span>.<span class="pl-en">_</span>
<span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">spark</span>.<span class="pl-en">sql</span>.<span class="pl-en">SaveMode</span>
@@ -302,10 +311,10 @@
.save()
spark.sql(
- s<span class="pl-s"><span class="pl-pds">"""</span></span>
+ <span class="pl-k">s</span><span class="pl-s">"""</span>
<span class="pl-s"> |SELECT country, sum(quantity), avg(price)</span>
<span class="pl-s"> | from sales GROUP BY country</span>
-<span class="pl-s"> <span class="pl-pds">"""</span></span>.stripMargin).show
+<span class="pl-s"> </span><span class="pl-s">"""</span>.stripMargin).show
spark.stop</pre></div>
<h4>
@@ -366,6 +375,11 @@ AS
FROM sales
GROUP BY country, sex
</code></pre>
+<p><strong>NOTE</strong>:</p>
+<ul>
+<li>Preaggregate datamap is deprecated and it is replaced by MV datamap.
+Please refer <a href="./mv-datamap-guide.html">CarbonData MV DataMap</a> for more info.</li>
+</ul>
<h4>
<a id="functions-supported-in-pre-aggregate-table" class="anchor" href="#functions-supported-in-pre-aggregate-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Functions supported in pre-aggregate table</h4>
<table>
diff --git a/content/presto-guide.html b/content/presto-guide.html
index b104bdb..af899f8 100644
--- a/content/presto-guide.html
+++ b/content/presto-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/quick-start-guide.html b/content/quick-start-guide.html
index 61cedd3..d3c788c 100644
--- a/content/quick-start-guide.html
+++ b/content/quick-start-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/release-guide.html b/content/release-guide.html
index bbb02b5..e034a28 100644
--- a/content/release-guide.html
+++ b/content/release-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/s3-guide.html b/content/s3-guide.html
index 41fd0ae..3c3c597 100644
--- a/content/s3-guide.html
+++ b/content/s3-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/sdk-guide.html b/content/sdk-guide.html
index 23dd148..3e7f999 100644
--- a/content/sdk-guide.html
+++ b/content/sdk-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -421,6 +430,11 @@ These SDK writer output contains just carbondata and carbonindex files. No metad
<td>DataTypes.STRING</td>
</tr>
<tr>
+<td>BINARY</td>
+<td>-</td>
+<td>DataTypes.BINARY</td>
+</tr>
+<tr>
<td>FLOAT</td>
<td>FLOAT</td>
<td>DataTypes.FLOAT</td>
@@ -511,7 +525,7 @@ c. Timestamp (microsecond precision)
The timestamp-micros logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one microsecond.
A timestamp-micros logical type annotates an Avro long, where the long stores the number of microseconds from the unix epoch, 1 January 1970 00:00:00.000000 UTC.
d. Decimal
-The decimal logical type represents an arbitrary-precision signed decimal number of the form unscaled � 10-scale.
+The decimal logical type represents an arbitrary-precision signed decimal number of the form <em>unscaled × 10<sup>-scale</sup></em>.
A decimal logical type annotates Avro bytes or fixed types. The byte array must contain the two's-complement representation of the unscaled integer value in big-endian byte order. The scale is fixed, and is specified using an attribute.
e. Time (millisecond precision)
The time-millis logical type represents a time of day, with no reference to a particular calendar, time zone or date, with a precision of one millisecond.
@@ -655,6 +669,7 @@ public CarbonWriterBuilder withLoadOptions(Map<String, String> options);
* default value is null.
* l. inverted_index -- comma separated string columns for which inverted index needs to be
* generated
+ * m. table_page_size_inmb -- [1-1755] MB.
*
* @return updated CarbonWriterBuilder
*/
@@ -850,6 +865,8 @@ while (reader.hasNext()) {
reader.close();
</code></pre>
<p>Find example code at <a href="https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java" target=_blank>CarbonReaderExample</a> in the CarbonData repo.</p>
+<p>SDK reader also supports reading carbondata files and filling it to apache arrow vectors.
+Find example code at <a href="https://github.com/apache/carbondata/blob/master/store/sdk/src/test/java/org/apache/carbondata/sdk/file/ArrowCarbonReaderTest.java" target=_blank>ArrowCarbonReaderTest</a> in the CarbonData repo.</p>
<h2>
<a id="api-list-1" class="anchor" href="#api-list-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>API List</h2>
<h3>
@@ -911,6 +928,72 @@ public Object[] readNextBatchRow();
public void close();
</code></pre>
<h3>
+<a id="class-orgapachecarbondatasdkfilearrowcarbonreader" class="anchor" href="#class-orgapachecarbondatasdkfilearrowcarbonreader" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.ArrowCarbonReader</h3>
+<pre><code>/**
+ * Carbon reader will fill the arrow vector after reading the carbondata files.
+ * This arrow byte[] can be used to create arrow table and used for in memory analytics
+ * Note: create a reader at blocklet level, so that arrow byte[] will not exceed INT_MAX
+ *
+ * @param carbonSchema org.apache.carbondata.sdk.file.Schema
+ * @return Serialized byte array
+ * @throws Exception
+ */
+public byte[] readArrowBatch(Schema carbonSchema) throws Exception;
+</code></pre>
+<pre><code>/**
+ * Carbon reader will fill the arrow vector after reading the carbondata files.
+ * This arrow byte[] can be used to create arrow table and used for in memory analytics
+ * Note: create a reader at blocklet level, so that arrow byte[] will not exceed INT_MAX
+ * User need to close the VectorSchemaRoot after usage by calling VectorSchemaRoot.close()
+ *
+ * @param carbonSchema org.apache.carbondata.sdk.file.Schema
+ * @return Arrow VectorSchemaRoot
+ * @throws Exception
+ */
+public VectorSchemaRoot readArrowVectors(Schema carbonSchema) throws Exception;
+</code></pre>
+<pre><code>/**
+ * Carbon reader will fill the arrow vector after reading carbondata files.
+ * Here unsafe memory address will be returned instead of byte[],
+ * so that this address can be sent across java to python or c modules and
+ * can directly read the content from this unsafe memory
+ * Note:Create a carbon reader at blocklet level using CarbonReader.buildWithSplits(split) method,
+ * so that arrow byte[] will not exceed INT_MAX.
+ *
+ * @param carbonSchema org.apache.carbondata.sdk.file.Schema
+ * @return address of the unsafe memory where arrow buffer is stored
+ * @throws Exception
+ */
+public long readArrowBatchAddress(Schema carbonSchema) throws Exception;
+</code></pre>
+<pre><code>/**
+ * Free the unsafe memory allocated , if unsafe arrow batch is used.
+ *
+ * @param address address of the unsafe memory where arrow bufferer is stored
+ */
+public void freeArrowBatchMemory(long address)
+</code></pre>
+<h3>
+<a id="class-orgapachecarbondatasdkfilearrowarrowconverter" class="anchor" href="#class-orgapachecarbondatasdkfilearrowarrowconverter" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.arrow.ArrowConverter</h3>
+<pre><code>/**
+ * To get the arrow vectors directly after filling from carbondata
+ *
+ * @return Arrow VectorSchemaRoot. which contains array of arrow vectors.
+ */
+public VectorSchemaRoot getArrowVectors() throws IOException;
+</code></pre>
+<pre><code>/**
+ * Utility API to convert back the arrow byte[] to arrow ArrowRecordBatch.
+ * User need to close the ArrowRecordBatch after usage by calling ArrowRecordBatch.close()
+ *
+ * @param batchBytes input byte array
+ * @param bufferAllocator arrow buffer allocator
+ * @return ArrowRecordBatch
+ * @throws IOException
+ */
+public static ArrowRecordBatch byteArrayToArrowBatch(byte[] batchBytes, BufferAllocator bufferAllocator) throws IOException;
+</code></pre>
+<h3>
<a id="class-orgapachecarbondatasdkfilecarbonreaderbuilder" class="anchor" href="#class-orgapachecarbondatasdkfilecarbonreaderbuilder" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.CarbonReaderBuilder</h3>
<pre><code>/**
* Construct a CarbonReaderBuilder with table path and table name
diff --git a/content/security.html b/content/security.html
index b9ba692..0cbb03f 100644
--- a/content/security.html
+++ b/content/security.html
@@ -45,6 +45,12 @@
aria-expanded="false">Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
diff --git a/content/segment-management-on-carbondata.html b/content/segment-management-on-carbondata.html
index b80b22c..bb0d793 100644
--- a/content/segment-management-on-carbondata.html
+++ b/content/segment-management-on-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/streaming-guide.html b/content/streaming-guide.html
index 7f3b414..55f9f76 100644
--- a/content/streaming-guide.html
+++ b/content/streaming-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -287,16 +296,16 @@
spark.sparkContext.setLogLevel(<span class="pl-s"><span class="pl-pds">"</span>ERROR<span class="pl-pds">"</span></span>)
<span class="pl-c"><span class="pl-c">//</span> drop table if exists previously</span>
- spark.sql(s<span class="pl-s"><span class="pl-pds">"</span>DROP TABLE IF EXISTS carbon_table<span class="pl-pds">"</span></span>)
+ spark.sql(<span class="pl-k">s</span><span class="pl-s">"</span><span class="pl-s">DROP TABLE IF EXISTS carbon_table</span><span class="pl-s">"</span>)
<span class="pl-c"><span class="pl-c">//</span> Create target carbon table and populate with initial data</span>
spark.sql(
- s<span class="pl-s"><span class="pl-pds">"""</span></span>
+ <span class="pl-k">s</span><span class="pl-s">"""</span>
<span class="pl-s"> | CREATE TABLE carbon_table (</span>
<span class="pl-s"> | col1 INT,</span>
<span class="pl-s"> | col2 STRING</span>
<span class="pl-s"> | )</span>
<span class="pl-s"> | STORED AS carbondata</span>
-<span class="pl-s"> | TBLPROPERTIES('streaming'='true')<span class="pl-pds">"""</span></span>.stripMargin)
+<span class="pl-s"> | TBLPROPERTIES('streaming'='true')</span><span class="pl-s">"""</span>.stripMargin)
<span class="pl-k">val</span> <span class="pl-smi">carbonTable</span> <span class="pl-k">=</span> <span class="pl-en">CarbonEnv</span>.getCarbonTable(<span class="pl-en">Some</span>(<span class="pl-s"><span class="pl-pds">"</span>default<span class="pl-pds">"</span></span>), <span class="pl-s"><span class="pl-pds">"</span>carbon_table<span class="pl-pds">"</span></span>)(spark)
<span class="pl-k">val</span> <span class="pl-smi">tablePath</span> <span class="pl-k">=</span> carbonTable.getTablePath
diff --git a/content/supported-data-types-in-carbondata.html b/content/supported-data-types-in-carbondata.html
index c00f28d..3c4a32c 100644
--- a/content/supported-data-types-in-carbondata.html
+++ b/content/supported-data-types-in-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -269,6 +278,7 @@ Please refer to TBLProperties in <a href="./ddl-of-carbondata.html#create-table"
<p>Other Types</p>
<ul>
<li>BOOLEAN</li>
+<li>BINARY</li>
</ul>
</li>
</ul>
diff --git a/content/timeseries-datamap-guide.html b/content/timeseries-datamap-guide.html
index b3770ee..a0bf828 100644
--- a/content/timeseries-datamap-guide.html
+++ b/content/timeseries-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/usecases.html b/content/usecases.html
index fa2c0ea..e22945f 100644
--- a/content/usecases.html
+++ b/content/usecases.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/content/videogallery.html b/content/videogallery.html
index 614d8c6..6eb4503 100644
--- a/content/videogallery.html
+++ b/content/videogallery.html
@@ -49,6 +49,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index 430fa47..eb1c327 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -21,12 +21,14 @@ fileList=["configuration-parameters",
"carbon-as-spark-datasource-guide",
"alluxio-guide",
"hive-guide",
+ "index-server",
"presto-guide"
]
dataMapFileList=[
"bloomfilter-datamap-guide",
"lucene-datamap-guide",
"preaggregate-datamap-guide",
+ "mv-datamap-guide",
"timeseries-datamap-guide",
"datamap-management"]
diff --git a/src/main/scala/html/header.html b/src/main/scala/html/header.html
index b8079c9..427274f 100644
--- a/src/main/scala/html/header.html
+++ b/src/main/scala/html/header.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/scala/scripts/index-server b/src/main/scala/scripts/index-server
new file mode 100644
index 0000000..c8c9d59
--- /dev/null
+++ b/src/main/scala/scripts/index-server
@@ -0,0 +1,4 @@
+<script>
+// Show selected style on nav item
+$(function() { $('.b-nav__indexserver').addClass('selected'); });
+</script>
\ No newline at end of file
diff --git a/src/main/scala/scripts/mv-datamap-guide b/src/main/scala/scripts/mv-datamap-guide
new file mode 100644
index 0000000..013400a
--- /dev/null
+++ b/src/main/scala/scripts/mv-datamap-guide
@@ -0,0 +1,11 @@
+<script>
+$(function() {
+ // Show selected style on nav item
+ $('.b-nav__datamap').addClass('selected');
+
+ if (!$('.b-nav__datamap').parent().hasClass('nav__item__with__subs--expanded')) {
+ // Display datamap subnav items
+ $('.b-nav__datamap').parent().toggleClass('nav__item__with__subs--expanded');
+ }
+});
+</script>
\ No newline at end of file
diff --git a/src/main/webapp/CSDK-guide.html b/src/main/webapp/CSDK-guide.html
index a7f2b76..1c252a5 100644
--- a/src/main/webapp/CSDK-guide.html
+++ b/src/main/webapp/CSDK-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/alluxio-guide.html b/src/main/webapp/alluxio-guide.html
index b6bcdd7..b148273 100644
--- a/src/main/webapp/alluxio-guide.html
+++ b/src/main/webapp/alluxio-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/bloomfilter-datamap-guide.html b/src/main/webapp/bloomfilter-datamap-guide.html
index cc7ee14..51682cc 100644
--- a/src/main/webapp/bloomfilter-datamap-guide.html
+++ b/src/main/webapp/bloomfilter-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/carbon-as-spark-datasource-guide.html b/src/main/webapp/carbon-as-spark-datasource-guide.html
index 9c8224a..d19bfe9 100644
--- a/src/main/webapp/carbon-as-spark-datasource-guide.html
+++ b/src/main/webapp/carbon-as-spark-datasource-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -221,7 +230,7 @@
<h1>
<a id="carbondata-as-sparks-datasource" class="anchor" href="#carbondata-as-sparks-datasource" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData as Spark's Datasource</h1>
<p>The CarbonData fileformat is now integrated as Spark datasource for read and write operation without using CarbonSession. This is useful for users who wants to use carbondata as spark's data source.</p>
-<p><strong>Note:</strong> You can only apply the functions/features supported by spark datasource APIs, functionalities supported would be similar to Parquet. The carbon session features are not supported.</p>
+<p><strong>Note:</strong> You can only apply the functions/features supported by spark datasource APIs, functionalities supported would be similar to Parquet. The carbon session features are not supported. The result is displayed as byte array format when select query on binary column in spark-sql.</p>
<h1>
<a id="create-table-with-ddl" class="anchor" href="#create-table-with-ddl" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create Table with DDL</h1>
<p>Now you can create Carbon table using Spark's datasource DDL syntax.</p>
@@ -250,7 +259,7 @@
<tr>
<td>table_blocksize</td>
<td>1024</td>
-<td>Size of blocks to write onto hdfs. For more details, see <a href="./ddl-of-carbondata.html#table-block-size-configuration">Table Block Size Configuration</a>.</td>
+<td>Size of blocks to write onto hdfs. For more details, see <a href="./ddl-of-carbondata.html#table-block-size-configuration">Table Block Size Configuration</a>.</td>
</tr>
<tr>
<td>table_blocklet_size</td>
@@ -258,35 +267,41 @@
<td>Size of blocklet to write.</td>
</tr>
<tr>
+<td>table_page_size_inmb</td>
+<td>0</td>
+<td>Size of each page in carbon table, if page size crosses this value before 32000 rows, page will be cut to that many rows. Helps in keep page size to fit cache size</td>
+</tr>
+<tr>
<td>local_dictionary_threshold</td>
<td>10000</td>
-<td>Cardinality upto which the local dictionary can be generated. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
+<td>Cardinality upto which the local dictionary can be generated. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
</tr>
<tr>
<td>local_dictionary_enable</td>
<td>false</td>
-<td>Enable local dictionary generation. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
+<td>Enable local dictionary generation. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
</tr>
<tr>
<td>sort_columns</td>
<td>all dimensions are sorted</td>
-<td>Columns to include in sort and its order of sort. For more details, see <a href="./ddl-of-carbondata.html#sort-columns-configuration">Sort Columns Configuration</a>.</td>
+<td>Columns to include in sort and its order of sort. For more details, see <a href="./ddl-of-carbondata.html#sort-columns-configuration">Sort Columns Configuration</a>.</td>
</tr>
<tr>
<td>sort_scope</td>
<td>local_sort</td>
-<td>Sort scope of the load.Options include no sort, local sort, batch sort, and global sort. For more details, see <a href="./ddl-of-carbondata.html#sort-scope-configuration">Sort Scope Configuration</a>.</td>
+<td>Sort scope of the load.Options include no sort, local sort, batch sort, and global sort. For more details, see <a href="./ddl-of-carbondata.html#sort-scope-configuration">Sort Scope Configuration</a>.</td>
</tr>
<tr>
<td>long_string_columns</td>
<td>null</td>
-<td>Comma separated string/char/varchar columns which are more than 32k length. For more details, see <a href="./ddl-of-carbondata.html#string-longer-than-32000-characters">String longer than 32000 characters</a>.</td>
+<td>Comma separated string/char/varchar columns which are more than 32k length. For more details, see <a href="./ddl-of-carbondata.html#string-longer-than-32000-characters">String longer than 32000 characters</a>.</td>
</tr>
</tbody>
</table>
+<p><strong>NOTE:</strong> please set long_string_columns for varchar column.</p>
<h2>
<a id="example" class="anchor" href="#example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example</h2>
-<pre><code> CREATE TABLE CARBON_TABLE (NAME STRING) USING CARBON OPTIONS('table_block_size'='256')
+<pre><code> CREATE TABLE CARBON_TABLE (NAME STRING) USING CARBON OPTIONS('table_block_size'='256')
</code></pre>
<h1>
<a id="using-dataframe" class="anchor" href="#using-dataframe" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Using DataFrame</h1>
diff --git a/src/main/webapp/configuration-parameters.html b/src/main/webapp/configuration-parameters.html
index ca4b9e9..776d64a 100644
--- a/src/main/webapp/configuration-parameters.html
+++ b/src/main/webapp/configuration-parameters.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -326,6 +335,11 @@
<td>5</td>
<td>Specifies the interval between the retries to obtain the lock for any operation other than load. <strong>NOTE:</strong> Refer to <em><strong>carbon.lock.retries</strong></em> for understanding why CarbonData uses locks for operations.</td>
</tr>
+<tr>
+<td>carbon.fs.custom.file.provider</td>
+<td>None</td>
+<td>To support FileTypeInterface for configuring custom CarbonFile implementation to work with custom FileSystem.</td>
+</tr>
</tbody>
</table>
<h2>
@@ -540,6 +554,16 @@
<td>200</td>
<td>CarbonData will write the min max values for string/varchar types column using the byte count specified by this configuration. Max value is 1000 bytes(500 characters) and Min value is 10 bytes(5 characters). <strong>NOTE:</strong> This property is useful for reducing the store size thereby improving the query performance but can lead to query degradation if value is not configured properly.</td>
</tr>
+<tr>
+<td>carbon.merge.index.failure.throw.exception</td>
+<td>true</td>
+<td>It is used to configure whether or not merge index failure should result in data load failure also.</td>
+</tr>
+<tr>
+<td>carbon.binary.decoder</td>
+<td>None</td>
+<td>Support configurable decode for loading. Two decoders supported: base64 and hex</td>
+</tr>
</tbody>
</table>
<h2>
@@ -624,6 +648,11 @@
<td>true</td>
<td>Each CarbonData file has a companion CarbonIndex file which maintains the metadata about the data. These CarbonIndex files are read and loaded into driver and is used subsequently for pruning of data during queries. These CarbonIndex files are very small in size(few KB) and are many. Reading many small files from HDFS is not efficient and leads to slow IO performance. Hence these CarbonIndex files belonging to a segment can be combined into a single file and read once there by incre [...]
</tr>
+<tr>
+<td>carbon.enable.range.compaction</td>
+<td>true</td>
+<td>To configure Ranges-based Compaction to be used or not for RANGE_COLUMN. If true after compaction also the data would be present in ranges.</td>
+</tr>
</tbody>
</table>
<h2>
@@ -743,6 +772,11 @@
<td>false</td>
<td>When enabled complete row filters will be handled by carbon in case of vector. If it is disabled then only page level pruning will be done by carbon and row level filtering will be done by spark for vector. And also there are scan optimizations in carbon to avoid multiple data copies when this parameter is set to false. There is no change in flow for non-vector based queries.</td>
</tr>
+<tr>
+<td>carbon.query.prefetch.enable</td>
+<td>true</td>
+<td>By default this property is true, so prefetch is used in query to read next blocklet asynchronously in other thread while processing current blocklet in main thread. This can help to reduce CPU idle time. Setting this property false will disable this prefetch feature in query.</td>
+</tr>
</tbody>
</table>
<h2>
diff --git a/src/main/webapp/datamap-developer-guide.html b/src/main/webapp/datamap-developer-guide.html
index e5ec135..09da4cc 100644
--- a/src/main/webapp/datamap-developer-guide.html
+++ b/src/main/webapp/datamap-developer-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/datamap-management.html b/src/main/webapp/datamap-management.html
index 172dc00..a4d7461 100644
--- a/src/main/webapp/datamap-management.html
+++ b/src/main/webapp/datamap-management.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -274,7 +283,7 @@ AS
<td>mv</td>
<td>multi-table pre-aggregate table</td>
<td>No DMPROPERTY is required</td>
-<td>Manual</td>
+<td>Manual/Automatic</td>
</tr>
<tr>
<td>lucene</td>
@@ -297,8 +306,6 @@ AS
<li>Automatic Refresh: Create datamap without <code>WITH DEFERRED REBUILD</code> in the statement, which is by default.</li>
<li>Manual Refresh: Create datamap with <code>WITH DEFERRED REBUILD</code> in the statement</li>
</ol>
-<p><strong>CAUTION:</strong>
-If user create MV datamap without specifying <code>WITH DEFERRED REBUILD</code>, carbondata will give a warning and treat the datamap as deferred rebuild.</p>
<h3>
<a id="automatic-refresh" class="anchor" href="#automatic-refresh" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Automatic Refresh</h3>
<p>When user creates a datamap on the main table without using <code>WITH DEFERRED REBUILD</code> syntax, the datamap will be managed by system automatically.
@@ -361,6 +368,9 @@ Scan Table: default.datamap1_table
<li>DataMapProviderName like mv, preaggreagte, timeseries, etc</li>
<li>Associated Table</li>
<li>DataMap Properties</li>
+<li>DataMap status (ENABLED/DISABLED)</li>
+<li>Sync Status - which displays Last segment Id of main table synced with datamap table and its load
+end time (Applicable only for mv datamap)</li>
</ul>
<h3>
<a id="compaction-on-datamap" class="anchor" href="#compaction-on-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compaction on DataMap</h3>
diff --git a/src/main/webapp/ddl-of-carbondata.html b/src/main/webapp/ddl-of-carbondata.html
index 8dea391..5720f10 100644
--- a/src/main/webapp/ddl-of-carbondata.html
+++ b/src/main/webapp/ddl-of-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -356,6 +365,10 @@ STORED AS carbondata
<td>Size of blocklet to write in the file</td>
</tr>
<tr>
+<td><a href="#table-page-size-configuration">TABLE_PAGE_SIZE_INMB</a></td>
+<td>Size of page in MB; if page size crosses this value before 32000 rows, page will be cut to this many rows and remaining rows are processed in the subsequent pages. This helps in keeping page size to fit in cpu cache size</td>
+</tr>
+<tr>
<td><a href="#table-compaction-configuration">MAJOR_COMPACTION_SIZE</a></td>
<td>Size upto which the segments can be combined into one</td>
</tr>
@@ -440,7 +453,7 @@ Suggested use cases : do dictionary encoding for low cardinality columns, it mig
</code></pre>
<p><strong>NOTE</strong>:</p>
<ul>
-<li>Dictionary Include/Exclude for complex child columns is not supported.</li>
+<li>Dictionary Include/Exclude for complex child columns is not supported. Dictionary Include doesn't support binary data type.</li>
<li>Dictionary is global. Except global dictionary, there are local dictionary and non-dictionary in CarbonData.</li>
</ul>
</li>
@@ -480,6 +493,7 @@ Suggested use cases : do dictionary encoding for low cardinality columns, it mig
<li>BOOLEAN</li>
<li>FLOAT</li>
<li>BYTE</li>
+<li>Binary</li>
</ul>
</li>
<li>
@@ -498,8 +512,8 @@ Suggested use cases : do dictionary encoding for low cardinality columns, it mig
<tbody>
<tr>
<td>carbon.local.dictionary.enable</td>
-<td>false</td>
-<td>By default, Local Dictionary will be disabled for the carbondata table.</td>
+<td>true</td>
+<td>By default, Local Dictionary will be enabled for the carbondata table.</td>
</tr>
<tr>
<td>carbon.local.dictionary.decoder.fallback</td>
@@ -589,14 +603,14 @@ Suggested use cases : For high cardinality columns, you can disable the inverted
<a id="sort-columns-configuration" class="anchor" href="#sort-columns-configuration" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Sort Columns Configuration</h5>
<p>This property is for users to specify which columns belong to the MDK(Multi-Dimensions-Key) index.</p>
<ul>
-<li>If users don't specify "SORT_COLUMN" property, by default no columns are sorted</li>
+<li>If users don't specify "SORT_COLUMNS" property, by default no columns are sorted</li>
<li>If this property is specified but with empty argument, then the table will be loaded without sort.</li>
<li>This supports only string, date, timestamp, short, int, long, byte and boolean data types.
Suggested use cases : Only build MDK index for required columns,it might help to improve the data loading performance.</li>
</ul>
<pre><code>TBLPROPERTIES ('SORT_COLUMNS'='column1, column3')
</code></pre>
-<p><strong>NOTE</strong>: Sort_Columns for Complex datatype columns is not supported.</p>
+<p><strong>NOTE</strong>: Sort_Columns for Complex datatype columns and binary data type is not supported.</p>
</li>
<li>
<h5>
@@ -647,6 +661,21 @@ Blocklet is the minimum IO read unit, in case of point queries reduce blocklet s
</li>
<li>
<h5>
+<a id="table-page-size-configuration" class="anchor" href="#table-page-size-configuration" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Table page Size Configuration</h5>
+<p>This property is for setting page size in the carbondata file
+and supports a range of 1 MB to 1755 MB.
+If page size crosses this value before 32000 rows, page will be cut to that many rows.
+Helps in keeping page size to fit cpu cache size.</p>
+<p>This property can be configured if the table has string, varchar, binary or complex datatype columns.
+Because for these columns 32000 rows in one page may exceed 1755 MB and snappy compression will fail in that scenario.
+Also if page size is huge, page cannot be fit in CPU cache.
+So, configuring smaller values of this property (say 1 MB) can result in better use of CPU cache for pages.</p>
+<p>Example usage:</p>
+<pre><code>TBLPROPERTIES ('TABLE_PAGE_SIZE_INMB'='5')
+</code></pre>
+</li>
+<li>
+<h5>
<a id="table-compaction-configuration" class="anchor" href="#table-compaction-configuration" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Table Compaction Configuration</h5>
<p>These properties are table level compaction configurations, if not specified, system level configurations in carbon.properties will be used.
Following are 5 configurations:</p>
@@ -674,7 +703,7 @@ Following are 5 configurations:</p>
<li>
<h5>
<a id="caching-minmax-value-for-required-columns" class="anchor" href="#caching-minmax-value-for-required-columns" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Caching Min/Max Value for Required Columns</h5>
-<p>By default, CarbonData caches min and max values of all the columns in schema. As the load increases, the memory required to hold the min and max values increases considerably. This feature enables you to configure min and max values only for the required columns, resulting in optimized memory usage.</p>
+<p>By default, CarbonData caches min and max values of all the columns in schema. As the load increases, the memory required to hold the min and max values increases considerably. This feature enables you to configure min and max values only for the required columns, resulting in optimized memory usage. This feature doesn't support binary data type.</p>
<p>Following are the valid values for COLUMN_META_CACHE:</p>
<ul>
<li>If you want no column min/max values to be cached in the driver.</li>
@@ -813,7 +842,8 @@ Notice that once you enable this feature, for load balance, carbondata will igno
<h5>
<a id="range-column" class="anchor" href="#range-column" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Range Column</h5>
<p>This property is used to specify a column to partition the input data by range.
-Only one column can be configured. During data loading, you can use "global_sort_partitions" or "scale_factor" to avoid generating small files.</p>
+Only one column can be configured. During data loading, you can use "global_sort_partitions" or "scale_factor" to avoid generating small files.
+This feature doesn't support binary data type.</p>
<pre><code>TBLPROPERTIES('RANGE_COLUMN'='col1')
</code></pre>
</li>
@@ -1048,6 +1078,28 @@ This can be achieved by using the alter table set command.</p>
<pre><code>ALTER TABLE tablename UNSET TBLPROPERTIES('SORT_SCOPE')
</code></pre>
</li>
+<li>
+<h5>
+<a id="sort-columns" class="anchor" href="#sort-columns" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>SORT COLUMNS</h5>
+<p>Example to SET SORT COLUMNS:</p>
+<pre><code>ALTER TABLE tablename SET TBLPROPERTIES('SORT_COLUMNS'='column1')
+</code></pre>
+<p>After this operation, the new loading will use the new SORT_COLUMNS. The user can adjust
+the SORT_COLUMNS according to the query, but it will not impact the old data directly. So
+it will not impact the query performance of the old data segments which are not sorted by
+new SORT_COLUMNS.</p>
+<p>UNSET is not supported, but it can set SORT_COLUMNS to empty string instead of using UNSET.
+NOTE: When SORT_SCOPE is not NO_SORT, then setting SORT_COLUMNS to empty string is not valid.</p>
+<pre><code>ALTER TABLE tablename SET TBLPROPERTIES('SORT_COLUMNS'='')
+</code></pre>
+<p><strong>NOTE:</strong></p>
+<ul>
+<li>The future version will enhance "custom" compaction to sort the old segment one by one.</li>
+<li>The streaming table is not supported for SORT_COLUMNS modification.</li>
+<li>If the inverted index columns are removed from the new SORT_COLUMNS, they will not
+create the inverted index. But the old configuration of INVERTED_INDEX will be kept.</li>
+</ul>
+</li>
</ul>
</li>
</ul>
@@ -1124,7 +1176,7 @@ TBLPROPERTIES ('DICTIONARY_INCLUDE'='productNumber')
PARTITIONED BY (productCategory STRING, productBatch STRING)
STORED AS carbondata
</code></pre>
-<p><strong>NOTE:</strong> Hive partition is not supported on complex datatype columns.</p>
+<p><strong>NOTE:</strong> Hive partition is not supported on complex data type columns.</p>
<h4>
<a id="show-partitions" class="anchor" href="#show-partitions" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Show Partitions</h4>
<p>This command gets the Hive partition information of the table</p>
diff --git a/src/main/webapp/dml-of-carbondata.html b/src/main/webapp/dml-of-carbondata.html
index eaf4476..05f848c 100644
--- a/src/main/webapp/dml-of-carbondata.html
+++ b/src/main/webapp/dml-of-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -343,6 +352,10 @@ OPTIONS(property_name=property_value, ...)
<td><a href="#scale_factor">SCALE_FACTOR</a></td>
<td>Control the partition size for RANGE_COLUMN feature</td>
</tr>
+<tr>
+<td>[CARBON_OPTIONS_BINARY_DECODER]</td>
+<td>Support configurable decode for loading from csv</td>
+</tr>
</tbody>
</table>
<ul>
@@ -550,7 +563,9 @@ OPTIONS('BAD_RECORDS_LOGGER_ENABLE'='true','BAD_RECORD_PATH'='hdfs://hacluster/t
<h5>
<a id="global_sort_partitions" class="anchor" href="#global_sort_partitions" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>GLOBAL_SORT_PARTITIONS:</h5>
<p>If the SORT_SCOPE is defined as GLOBAL_SORT, then user can specify the number of partitions to use while shuffling data for sort using GLOBAL_SORT_PARTITIONS. If it is not configured, or configured less than 1, then it uses the number of map task as reduce task. It is recommended that each reduce task deal with 512MB-1GB data.
-For RANGE_COLUMN, GLOBAL_SORT_PARTITIONS is used to specify the number of range partitions also.</p>
+For RANGE_COLUMN, GLOBAL_SORT_PARTITIONS is used to specify the number of range partitions also.
+GLOBAL_SORT_PARTITIONS should be specified optimally during RANGE_COLUMN LOAD because if a higher number is configured then the load time may be less but it will result in creation of more files which would degrade the query and compaction performance.
+Conversely, if less partitions are configured then the load performance may degrade due to less use of parallelism but the query and compaction will become faster. Hence the user may choose optimal number depending on the use case.</p>
</li>
</ul>
<pre><code>OPTIONS('GLOBAL_SORT_PARTITIONS'='2')
@@ -578,6 +593,14 @@ For RANGE_COLUMN, GLOBAL_SORT_PARTITIONS is used to specify the number of range
<li>If both GLOBAL_SORT_PARTITIONS and SCALE_FACTOR are used at the same time, only GLOBAL_SORT_PARTITIONS is valid.</li>
<li>The compaction on RANGE_COLUMN will use LOCAL_SORT by default.</li>
</ul>
+<ul>
+<li>
+<h5>
+<a id="carbon_enable_range_compaction" class="anchor" href="#carbon_enable_range_compaction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CARBON_ENABLE_RANGE_COMPACTION</h5>
+</li>
+</ul>
+<p>To configure Ranges-based Compaction to be used or not for RANGE_COLUMN.
+The default value is 'true'.</p>
<h3>
diff --git a/src/main/webapp/documentation.html b/src/main/webapp/documentation.html
index b502f10..3ca010d 100644
--- a/src/main/webapp/documentation.html
+++ b/src/main/webapp/documentation.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/faq.html b/src/main/webapp/faq.html
index 49b793c..579b867 100644
--- a/src/main/webapp/faq.html
+++ b/src/main/webapp/faq.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/file-structure-of-carbondata.html b/src/main/webapp/file-structure-of-carbondata.html
index 10518af..e4975d4 100644
--- a/src/main/webapp/file-structure-of-carbondata.html
+++ b/src/main/webapp/file-structure-of-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/hive-guide.html b/src/main/webapp/hive-guide.html
index 1c11ed7..296b959 100644
--- a/src/main/webapp/hive-guide.html
+++ b/src/main/webapp/hive-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -222,9 +231,6 @@
<a id="quick-start" class="anchor" href="#quick-start" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Quick Start</h1>
<p>This tutorial provides a quick introduction to using current integration/hive module.</p>
<h2>
-<a id="build-in-120-hive-integration-only-support-spark21-and-hadoop272" class="anchor" href="#build-in-120-hive-integration-only-support-spark21-and-hadoop272" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Build (In 1.2.0, hive integration only support spark2.1 and hadoop2.7.2)</h2>
-<p>mvn -DskipTests -Pspark-2.1 -Phadoop-2.7.2 clean package</p>
-<h2>
<a id="prepare-carbondata-in-spark" class="anchor" href="#prepare-carbondata-in-spark" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Prepare CarbonData in Spark</h2>
<ul>
<li>
@@ -285,18 +291,41 @@ export HIVE_AUX_JARS_PATH=hive/auxlibs/
export HADOOP_OPTS="-Dorg.xerial.snappy.lib.path=/Library/Java/Extensions -Dorg.xerial.snappy.lib.name=libsnappyjava.jnilib -Dorg.xerial.snappy.tempdir=/Users/apple/DEMO/tmp"
</code></pre>
<h3>
-<a id="start-hive-client" class="anchor" href="#start-hive-client" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Start hive client</h3>
-<pre><code>$HIVE_HOME/bin/hive
+<a id="carbon-jars-to-be-placed" class="anchor" href="#carbon-jars-to-be-placed" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Carbon Jars to be placed</h3>
+<pre><code>hive/lib/ (for hive server)
+yarn/lib/ (for MapReduce)
+
+Carbon Jars to be copied to the above paths.
</code></pre>
<h3>
-<a id="query-data-from-hive-table" class="anchor" href="#query-data-from-hive-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Query data from hive table</h3>
+<a id="start-hive-beeline-to-query" class="anchor" href="#start-hive-beeline-to-query" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Start hive beeline to query</h3>
+<pre><code>$HIVE_HOME/bin/beeline
+</code></pre>
+<h3>
+<a id="query-data-from-hive" class="anchor" href="#query-data-from-hive" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Query data from hive</h3>
+<ul>
+<li>This is to read the carbon table through Hive. It is the integration of the carbon with Hive.</li>
+</ul>
<pre><code>set hive.mapred.supports.subdirectories=true;
-set mapreduce.input.fileinputformat.input.dir.recursive=true;
+set mapreduce.dir.recursive=true;
+These properties helps to recursively traverse through the directories to read the carbon folder structure.
+</code></pre>
+<h3>
+<a id="example" class="anchor" href="#example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example</h3>
+<pre><code> - In case if the carbon table is not set with the SERDE and the INPUTFORMAT/OUTPUTFORMAT, user can create a new hive managed table like below with the required details for the hive to read.
+create table hive_carbon_1(id int, name string, scale decimal, country string, salary double) ROW FORMAT SERDE 'org.apache.carbondata.hive.CarbonHiveSerDe' WITH SERDEPROPERTIES ('mapreduce.input.carboninputformat.databaseName'='default', 'mapreduce.input.carboninputformat.tableName'='HIVE_CARBON_EXAMPLE') STORED AS INPUTFORMAT 'org.apache.carbondata.hive.MapredCarbonInputFormat' OUTPUTFORMAT 'org.apache.carbondata.hive.MapredCarbonOutputFormat' LOCATION 'location_to_the_carbon_table';
-select * from hive_carbon;
-select count(*) from hive_carbon;
-select * from hive_carbon order by id;
+ - Query the table
+select * from hive_carbon_1;
+select count(*) from hive_carbon_1;
+select * from hive_carbon_1 order by id;
</code></pre>
+<h3>
+<a id="note" class="anchor" href="#note" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Note</h3>
+<ul>
+<li>Partition table support is not handled</li>
+<li>Map data type is not supported</li>
+</ul>
<script>
// Show selected style on nav item
$(function() { $('.b-nav__quickstart').addClass('selected'); });
diff --git a/src/main/webapp/how-to-contribute-to-apache-carbondata.html b/src/main/webapp/how-to-contribute-to-apache-carbondata.html
index 92ef5e4..28139df 100644
--- a/src/main/webapp/how-to-contribute-to-apache-carbondata.html
+++ b/src/main/webapp/how-to-contribute-to-apache-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/index-server.html b/src/main/webapp/index-server.html
new file mode 100644
index 0000000..f7bea90
--- /dev/null
+++ b/src/main/webapp/index-server.html
@@ -0,0 +1,572 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'>
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <title>CarbonData</title>
+ <style>
+
+ </style>
+ <!-- Bootstrap -->
+
+ <link rel="stylesheet" href="css/bootstrap.min.css">
+ <link href="css/style.css" rel="stylesheet">
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="js/jquery.min.js"></script>
+ <script src="js/bootstrap.min.js"></script>
+ <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js"></script>
+
+
+</head>
+<body>
+<header>
+ <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper">
+ <div class="container">
+ <div class="navbar-header">
+ <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse"
+ class="navbar-toggle collapsed" type="button">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="index.html" class="logo">
+ <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/>
+ </a>
+ </div>
+ <div class="navbar-collapse collapse cd_navcontnt" id="navbar">
+ <ul class="nav navbar-nav navbar-right navlist-custom">
+ <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a>
+ </li>
+ <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false"> Download <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
+ target="_blank">Apache CarbonData 1.5.3</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.2/"
+ target="_blank">Apache CarbonData 1.5.2</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.1/"
+ target="_blank">Apache CarbonData 1.5.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
+ target="_blank">Release Archive</a></li>
+ </ul>
+ </li>
+ <li><a href="documentation.html" class="active">Documentation</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false">Community <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/how-to-contribute-to-apache-carbondata.md"
+ target="_blank">Contributing to CarbonData</a></li>
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"
+ target="_blank">Release Guide</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"
+ target="_blank">Project PMC and Committers</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"
+ target="_blank">CarbonData Meetups</a></li>
+ <li><a href="security.html">Apache CarbonData Security</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/CARBONDATA" target="_blank">Apache
+ Jira</a></li>
+ <li><a href="videogallery.html">CarbonData Videos </a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="apache_link hidden-xs dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="hidden-lg hidden-md hidden-sm dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li>
+ <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a>
+
+ </li>
+
+ </ul>
+ </div><!--/.nav-collapse -->
+ <div id="search-box">
+ <form method="get" action="http://www.google.com/search" target="_blank">
+ <div class="search-block">
+ <table border="0" cellpadding="0" width="100%">
+ <tr>
+ <td style="width:80%">
+ <input type="text" name="q" size=" 5" maxlength="255" value=""
+ class="search-input" placeholder="Search...." required/>
+ </td>
+ <td style="width:20%">
+ <input type="submit" value="Search"/></td>
+ </tr>
+ <tr>
+ <td align="left" style="font-size:75%" colspan="2">
+ <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/>
+ <span style=" position: relative; top: -3px;"> Only search for CarbonData</span>
+ </td>
+ </tr>
+ </table>
+ </div>
+ </form>
+ </div>
+ </div>
+ </nav>
+</header> <!-- end Header part -->
+
+<div class="fixed-padding"></div> <!-- top padding with fixde header -->
+
+<section><!-- Dashboard nav -->
+ <div class="container-fluid q">
+ <div class="col-sm-12 col-md-12 maindashboard">
+ <div class="verticalnavbar">
+ <nav class="b-sticky-nav">
+ <div class="nav-scroller">
+ <div class="nav__inner">
+ <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a>
+ <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a>
+ <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a>
+ <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a>
+ <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a>
+ <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a>
+ <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a>
+ <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a>
+ <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a>
+ </div>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a>
+ <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a>
+ <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
+ <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
+ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
+ </div>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
+ <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
+ <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
+ <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
+ <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
+ <a class="b-nav__security nav__item" href="./security.html">Security</a>
+ <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a>
+ </div>
+ </div>
+ <div class="navindicator">
+ <div class="b-nav__intro navindicator__item"></div>
+ <div class="b-nav__quickstart navindicator__item"></div>
+ <div class="b-nav__uses navindicator__item"></div>
+ <div class="b-nav__docs navindicator__item"></div>
+ <div class="b-nav__datamap navindicator__item"></div>
+ <div class="b-nav__api navindicator__item"></div>
+ <div class="b-nav__perf navindicator__item"></div>
+ <div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
+ <div class="b-nav__faq navindicator__item"></div>
+ <div class="b-nav__contri navindicator__item"></div>
+ <div class="b-nav__security navindicator__item"></div>
+ </div>
+ </nav>
+ </div>
+ <div class="mdcontent">
+ <section>
+ <div style="padding:10px 15px;">
+ <div id="viewpage" name="viewpage">
+ <div class="row">
+ <div class="col-sm-12 col-md-12">
+ <div>
+<h1>
+<a id="distributed-index-server" class="anchor" href="#distributed-index-server" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Distributed Index Server</h1>
+<h2>
+<a id="background" class="anchor" href="#background" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Background</h2>
+<p>Carbon currently prunes and caches all block/blocklet datamap index information into the driver for
+normal table, for Bloom/Index datamaps the JDBC driver will launch a job to prune and cache the
+datamaps in executors.</p>
+<p>This causes the driver to become a bottleneck in the following ways:</p>
+<ol>
+<li>If the cache size becomes huge(70-80% of the driver memory) then there can be excessive GC in
+the driver which can slow down the query and the driver may even go OutOfMemory.</li>
+<li>LRU has to evict a lot of elements from the cache to accommodate the new objects which would
+in turn slow down the queries.</li>
+<li>For bloom there is no guarantee that the next query goes to the same executor to reuse the cache
+and hence cache could be duplicated in multiple executors.</li>
+<li>Multiple JDBC drivers need to maintain their own copy of the cache.</li>
+</ol>
+<p>Distributed Index Cache Server aims to solve the above mentioned problems.</p>
+<h2>
+<a id="distribution" class="anchor" href="#distribution" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Distribution</h2>
+<p>When enabled, any query on a carbon table will be routed to the index server service in form of
+a request. The request will consist of the table name, segments, filter expression and other
+information used for pruning.</p>
+<p>In IndexServer service a pruning RDD is fired which will take care of the pruning for that
+request. This RDD will be creating tasks based on the number of segments that are applicable for
+pruning. It can happen that the user has specified segments to access for that table, so only the
+specified segments would be applicable for pruning. Refer: <a href="https://github.com/apache/carbondata/blob/6e50c1c6fc1d6e82a4faf6dc6e0824299786ccc0/docs/segment-management-on-carbondata.html#query-data-with-specified-segments">query-data-with-specified-segments</a>.
+IndexServer driver would have 2 important tasks, distributing the segments equally among the
+available executors and keeping track of the executor where the segment is cached.</p>
+<p>To achieve this 2 separate mappings would be maintained as follows.</p>
+<ol>
+<li>
+<p>segment to executor location:
+This mapping will be maintained for each table and will enable the index server to track the
+cache location for each segment.</p>
+</li>
+<li>
+<p>Cache size held by each executor:
+This mapping will be used to distribute the segments equally(on the basis of size) among the
+executors.</p>
+</li>
+</ol>
+<p>Once a request is received each segment would be iterated over and
+checked against tableToExecutorMapping to find if a executor is already
+assigned. If a mapping already exists then it means that most
+probably(if not evicted by LRU) the segment is already cached in that
+executor and the task for that segment has to be fired on this executor.</p>
+<p>If mapping is not found then first check executorToCacheMapping against
+the available executor list to find if any unassigned executor is
+present and use that executor for the current segment. If all the
+executors are assigned with some segment then find the least loaded
+executor on the basis of size.</p>
+<p>Initially the segment index size would be used to distribute the
+segments fairly among the executor because the actual cache size would
+be known to the driver only when the segments are cached and appropriate
+information is returned to the driver.</p>
+<p><strong>NOTE:</strong> In case of legacy segment(version: 1.1) the index size is not available
+therefore all the legacy segments would be processed in a round robin
+fashion.</p>
+<p>After the job is completed the tasks would return the cache size held by
+each executor which would be updated to the executorToCacheMapping and
+the pruned blocklets which would be further used for result fetching.</p>
+<p><strong>Note:</strong> Multiple JDBC drivers can connect to the index server to use the cache.</p>
+<h2>
+<a id="reallocation-of-executor" class="anchor" href="#reallocation-of-executor" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Reallocation of executor</h2>
+<p>In case executor(s) become dead/unavailable then the segments that were
+earlier being handled by those would be reassigned to some other
+executor using the distribution logic.</p>
+<p><strong>Note:</strong> Cache loading would be done again in the new executor for the
+current query.</p>
+<h2>
+<a id="metacache-ddl" class="anchor" href="#metacache-ddl" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>MetaCache DDL</h2>
+<p>The show metacache DDL has a new column called cache location will indicate whether the cache is
+from executor or driver. To drop cache the user has to enable/disable the index server using the
+dynamic configuration to clear the cache of the desired location.</p>
+<p>Refer: <a href="https://github.com/apache/carbondata/blob/master/docs/ddl-of-carbondata.html#cache">MetaCacheDDL</a></p>
+<h2>
+<a id="fallback" class="anchor" href="#fallback" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Fallback</h2>
+<p>In case of any failure the index server would fallback to embedded mode
+which means that the JDBCServer would take care of distributed pruning.
+A similar job would be fired by the JDBCServer which would take care of
+pruning using its own executors. If for any reason the embedded mode
+also fails to prune the datamaps then the job would be passed on to
+driver.</p>
+<p><strong>NOTE:</strong> In case of embedded mode a job would be fired after pruning to clear the
+cache as data cached in JDBCServer executors would be of no use.</p>
+<h2>
+<a id="writing-splits-to-a-file" class="anchor" href="#writing-splits-to-a-file" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Writing splits to a file</h2>
+<p>If the response is too huge then it is better to write the splits to a file so that the driver can
+read this file and create the splits. This can be controlled using the property 'carbon.index.server
+.inmemory.serialization.threshold.inKB'. By default, the minimum value for this property is 0,
+meaning that no matter how small the splits are they would be written to the file. Maximum is
+102400KB which will mean if the size of the splits for a executor cross this value then they would
+be written to file.</p>
+<p>The user can set the location for these file by using 'carbon.indexserver.temp.path'. By default
+table path would be used to write the files.</p>
+<h2>
+<a id="security" class="anchor" href="#security" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Security</h2>
+<p>The security for the index server is controlled through 'spark.carbon.indexserver.keytab' and 'spark
+.carbon.indexserver.principal'. These allow the RPC framework to login using the principal. It is
+recommended that the principal should be a super user, and the user should be exclusive for index
+server so that it does not grant access to any other service. Internally the operations would be
+executed as a Privileged Action using the login user.</p>
+<p>The Index Server is a long running service therefore the 'spark.yarn.keytab' and 'spark.yarn
+.principal' should be configured.</p>
+<h2>
+<a id="configurations" class="anchor" href="#configurations" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Configurations</h2>
+<h5>
+<a id="carbonpropertiesjdbcserver" class="anchor" href="#carbonpropertiesjdbcserver" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>carbon.properties(JDBCServer)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">carbon.enable.index.server</td>
+<td align="center">false</td>
+<td align="center">Enable the use of index server for pruning for the whole application.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.ip</td>
+<td align="center">NA</td>
+<td align="center">Specify the IP/HOST on which the server is started. Better to specify the private IP.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.port</td>
+<td align="center">NA</td>
+<td align="center">The port on which the index server is started.</td>
+</tr>
+<tr>
+<td align="center">carbon.disable.index.server.fallback</td>
+<td align="center">false</td>
+<td align="center">Whether to enable/disable fallback for index server. Should be used for testing purposes only. Refer: <a href="#Fallback">Fallback</a>
+</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.max.jobname.length</td>
+<td align="center">NA</td>
+<td align="center">The max length of the job to show in the index server service UI. For bigger queries this may impact performance as the whole string would be sent from JDBCServer to IndexServer.</td>
+</tr>
+</tbody>
+</table>
+<h5>
+<a id="carbonpropertiesindexserver" class="anchor" href="#carbonpropertiesindexserver" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>carbon.properties(IndexServer)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">carbon.index.server.ip</td>
+<td align="center">NA</td>
+<td align="center">Specify the IP/HOST on which the server would be started. Better to specify the private IP.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.port</td>
+<td align="center">NA</td>
+<td align="center">The port on which the index server has to be started.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.max.worker.threads</td>
+<td align="center">500</td>
+<td align="center">Number of RPC handlers to open for accepting the requests from JDBC driver. Max accepted value is Integer.Max. Refer: <a href="https://github.com/apache/hive/blob/master/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java#L3441">Hive configuration</a>
+</td>
+</tr>
+<tr>
+<td align="center">carbon.max.executor.lru.cache.size</td>
+<td align="center">NA</td>
+<td align="center">Maximum memory <strong>(in MB)</strong> upto which the executor process can cache the data (DataMaps and reverse dictionary values). Only integer values greater than 0 are accepted. <strong>NOTE:</strong> Mandatory for the user to set.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.max.jobname.length</td>
+<td align="center">NA</td>
+<td align="center">The max length of the job to show in the index server application UI. For bigger queries this may impact performance as the whole string would be sent from JDBCServer to IndexServer.</td>
+</tr>
+<tr>
+<td align="center">carbon.max.executor.threads.for.block.pruning</td>
+<td align="center">4</td>
+<td align="center">max executor threads used for block pruning.</td>
+</tr>
+<tr>
+<td align="center">carbon.index.server.inmemory.serialization.threshold.inKB</td>
+<td align="center">300</td>
+<td align="center">Max in memory serialization size after reaching threshold data will be written to file. Min value that the user can set is 0KB and max is 102400KB.</td>
+</tr>
+<tr>
+<td align="center">carbon.indexserver.temp.path</td>
+<td align="center">tablePath</td>
+<td align="center">The folder to write the split files if in memory datamap size for network transfers crossed the 'carbon.index.server.inmemory.serialization.threshold.inKB' limit.</td>
+</tr>
+</tbody>
+</table>
+<h5>
+<a id="spark-defaultsconfonly-for-secure-mode" class="anchor" href="#spark-defaultsconfonly-for-secure-mode" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>spark-defaults.conf(only for secure mode)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">spark.carbon.indexserver.principal</td>
+<td align="center">NA</td>
+<td align="center">Used for authentication, whether a valid service is trying to connect to the server or not. Set in both IndexServer and JDBCServer.</td>
+</tr>
+<tr>
+<td align="center">spark.carbon.indexserver.keytab</td>
+<td align="center">NA</td>
+<td align="center">Specify the path to the keytab file through which authentication would happen. Set in both IndexServer and JDBCServer.</td>
+</tr>
+<tr>
+<td align="center">spark.dynamicAllocation.enabled</td>
+<td align="center">true</td>
+<td align="center">Set to false, so that spark does not kill the executor, If executors are killed, cache would be lost. Applicable only for Index Server.</td>
+</tr>
+<tr>
+<td align="center">spark.yarn.principal</td>
+<td align="center">NA</td>
+<td align="center">Should be set to the same user used for JDBCServer. Required only for IndexServer.</td>
+</tr>
+<tr>
+<td align="center">spark.yarn.keytab</td>
+<td align="center">NA</td>
+<td align="center">Should be set to the same as JDBCServer.</td>
+</tr>
+</tbody>
+</table>
+<h5>
+<a id="spark-defaultsconfnon-secure-mode" class="anchor" href="#spark-defaultsconfnon-secure-mode" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>spark-defaults.conf(non-secure mode)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">spark.dynamicAllocation.enabled</td>
+<td align="center">true</td>
+<td align="center">Set to false, so that spark does not kill the executor, If executors are killed, cache would be lost. Applicable only for Index Server.</td>
+</tr>
+</tbody>
+</table>
+<p><strong>NOTE:</strong> Its better to create a new user for indexserver principal,
+that will authenticate the user to access the index server and no other service.</p>
+<h5>
+<a id="core-sitexml" class="anchor" href="#core-sitexml" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>core-site.xml</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">ipc.client.rpc-timeout.ms</td>
+<td align="center">NA</td>
+<td align="center">Set the above property to some appropriate value based on your estimated query time. The best option is to set this to the same value as spark.network.timeout.</td>
+</tr>
+</tbody>
+</table>
+<h5>
+<a id="dynamic-propertiesset-command" class="anchor" href="#dynamic-propertiesset-command" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>dynamic-properties(set command)</h5>
+<table>
+<thead>
+<tr>
+<th align="center">Name</th>
+<th align="center">Default Value</th>
+<th align="center">Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">carbon.enable.index.server</td>
+<td align="center">false</td>
+<td align="center">Enable the use of index server for pruning for the current session.</td>
+</tr>
+<tr>
+<td align="center">carbon.enable.index.server.dbName.tableName</td>
+<td align="center">false</td>
+<td align="center">Enable the use of index server for the specified table in the current session.</td>
+</tr>
+</tbody>
+</table>
+<h2>
+<a id="starting-the-server" class="anchor" href="#starting-the-server" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Starting the Server</h2>
+<pre><code>./bin/spark-submit --master [yarn/local] --[o ptional parameters] --class org.apache.carbondata.indexserver.IndexServer [path to carbondata-spark2-<version>.jar]
+</code></pre>
+<p>Or</p>
+<pre><code>./sbin/start-indexserver.sh --master yarn --num-executors 2 /<absolute path>/carbondata-spark2-1.6.0.0100.jar
+</code></pre>
+<h2>
+<a id="faq" class="anchor" href="#faq" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>FAQ</h2>
+<p>Q. <strong>Index Server is throwing Large response size exception.</strong></p>
+<p>A. The exception would show the size of response it is trying to send over the
+network. Use ipc.maximum.response.length to a value bigger than the
+response size.</p>
+<p>Q. <strong>Index server is throwing Kerberos principal not set exception</strong></p>
+<p>A. Set spark.carbon.indexserver.principal to the correct principal in both IndexServer and
+JDBCServer configurations.</p>
+<p>Q. <strong>Unable to connect to index server</strong></p>
+<p>A. Check whether the carbon.properties configurations are set in JDBCServer as well as the index
+server.</p>
+<p>Q. <strong>IndexServer is throwing FileNotFoundException for index files.</strong></p>
+<p>A. Check whether the Index server and JDBCServer are connected to the
+same namenode or not. And the store should be shared by both</p>
+<p>Q. <strong>OutOfMemoryException in DirectMemoryBuffer</strong></p>
+<p>A. Increase -XX:MaxDirectMemorySize in driver.extraJavaOptions to
+accommodate the large response in driver.</p>
+<script>
+// Show selected style on nav item
+$(function() { $('.b-nav__indexserver').addClass('selected'); });
+</script></div>
+</div>
+</div>
+</div>
+<div class="doc-footer">
+ <a href="#top" class="scroll-top">Top</a>
+</div>
+</div>
+</section>
+</div>
+</div>
+</div>
+</section><!-- End systemblock part -->
+<script src="js/custom.js"></script>
+</body>
+</html>
\ No newline at end of file
diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html
index 6f7554d..437a6ff 100644
--- a/src/main/webapp/index.html
+++ b/src/main/webapp/index.html
@@ -54,6 +54,12 @@
class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
diff --git a/src/main/webapp/introduction.html b/src/main/webapp/introduction.html
index 8af7902..79731fa 100644
--- a/src/main/webapp/introduction.html
+++ b/src/main/webapp/introduction.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/language-manual.html b/src/main/webapp/language-manual.html
index 0b7bca3..6e3cfbb 100644
--- a/src/main/webapp/language-manual.html
+++ b/src/main/webapp/language-manual.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/lucene-datamap-guide.html b/src/main/webapp/lucene-datamap-guide.html
index 4f3290c..db90362 100644
--- a/src/main/webapp/lucene-datamap-guide.html
+++ b/src/main/webapp/lucene-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/lucene-datamap-guide.html b/src/main/webapp/mv-datamap-guide.html
similarity index 58%
copy from src/main/webapp/lucene-datamap-guide.html
copy to src/main/webapp/mv-datamap-guide.html
index 4f3290c..a6a3ee2 100644
--- a/src/main/webapp/lucene-datamap-guide.html
+++ b/src/main/webapp/mv-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -219,153 +228,202 @@
<div class="col-sm-12 col-md-12">
<div>
<h1>
-<a id="carbondata-lucene-datamap-alpha-feature" class="anchor" href="#carbondata-lucene-datamap-alpha-feature" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Lucene DataMap (Alpha Feature)</h1>
+<a id="carbondata-mv-datamap" class="anchor" href="#carbondata-mv-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData MV DataMap</h1>
<ul>
-<li><a href="#datamap-management">DataMap Management</a></li>
-<li><a href="#lucene-datamap-introduction">Lucene Datamap</a></li>
+<li><a href="#quick-example">Quick Example</a></li>
+<li><a href="#mv-datamap-introduction">MV DataMap</a></li>
<li><a href="#loading-data">Loading Data</a></li>
<li><a href="#querying-data">Querying Data</a></li>
-<li><a href="#data-management-with-lucene-datamap">Data Management</a></li>
+<li><a href="#compacting-mv-tables">Compaction</a></li>
+<li><a href="#data-management-with-mv-tables">Data Management</a></li>
</ul>
-<h4>
-<a id="datamap-management" class="anchor" href="#datamap-management" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>DataMap Management</h4>
-<p>Lucene DataMap can be created using following DDL</p>
-<pre><code>CREATE DATAMAP [IF NOT EXISTS] datamap_name
-ON TABLE main_table
-USING 'lucene'
-DMPROPERTIES ('index_columns'='city, name', ...)
-</code></pre>
-<p>DataMap can be dropped using following DDL:</p>
-<pre><code>DROP DATAMAP [IF EXISTS] datamap_name
-ON TABLE main_table
-</code></pre>
-<p>To show all DataMaps created, use:</p>
-<pre><code>SHOW DATAMAP
-ON TABLE main_table
+<h2>
+<a id="quick-example" class="anchor" href="#quick-example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Quick example</h2>
+<p>Start spark-sql in terminal and run the following queries,</p>
+<pre><code>CREATE TABLE maintable(a int, b string, c int) stored by 'carbondata';
+insert into maintable select 1, 'ab', 2;
+CREATE DATAMAP datamap_1 on table maintable as SELECT a, sum(b) from maintable group by a;
+SELECT a, sum(b) from maintable group by a;
+// NOTE: run explain query and check if query hits the datamap table from the plan
+EXPLAIN SELECT a, sum(b) from maintable group by a;
</code></pre>
-<p>It will show all DataMaps created on main table.</p>
<h2>
-<a id="lucene-datamap-introduction" class="anchor" href="#lucene-datamap-introduction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Lucene DataMap Introduction</h2>
-<p>Lucene is a high performance, full featured text search engine. Lucene is integrated to carbon as
-an index datamap and managed along with main tables by CarbonData. User can create lucene datamap
-to improve query performance on string columns which has content of more length. So, user can
-search tokenized word or pattern of it using lucene query on text content.</p>
-<p>For instance, main table called <strong>datamap_test</strong> which is defined as:</p>
-<pre><code>CREATE TABLE datamap_test (
- name string,
- age int,
- city string,
- country string)
+<a id="mv-datamap-introduction" class="anchor" href="#mv-datamap-introduction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>MV DataMap Introduction</h2>
+<p>MV tables are created as DataMaps and managed as tables internally by CarbonData. User can create
+limitless MV datamaps on a table to improve query performance provided the storage requirements
+and loading time is acceptable.</p>
+<p>MV datamap can be a lazy or a non-lazy datamap. Once MV datamaps are created, CarbonData's
+CarbonAnalyzer helps to select the most efficient MV datamap based on the user query and rewrite
+the SQL to select the data from MV datamap instead of main table. Since the data size of MV
+datamap is smaller and data is pre-processed, user queries are much faster.</p>
+<p>For instance, main table called <strong>sales</strong> which is defined as</p>
+<pre><code>CREATE TABLE sales (
+ order_time timestamp,
+ user_id string,
+ sex string,
+ country string,
+ quantity int,
+ price bigint)
STORED AS carbondata
</code></pre>
-<p>User can create Lucene datamap using the Create DataMap DDL:</p>
-<pre><code>CREATE DATAMAP dm
-ON TABLE datamap_test
-USING 'lucene'
-DMPROPERTIES ('INDEX_COLUMNS' = 'name, country',)
+<p>User can create MV tables using the Create DataMap DDL</p>
+<pre><code>CREATE DATAMAP agg_sales
+ON TABLE sales
+USING "MV"
+AS
+ SELECT country, sex, sum(quantity), avg(price)
+ FROM sales
+ GROUP BY country, sex
</code></pre>
-<p><strong>DMProperties</strong></p>
+<p><strong>NOTE</strong>:</p>
+<ul>
+<li>
+<p>Group by/Filter columns has to be provided in projection list while creating mv datamap</p>
+</li>
+<li>
+<p>If only single parent table is involved in mv datamap creation, then TableProperties of Parent table
+(if not present in a aggregate function like sum(col)) listed below will be
+inherited to datamap table</p>
<ol>
-<li>INDEX_COLUMNS: The list of string columns on which lucene creates indexes.</li>
-<li>FLUSH_CACHE: size of the cache to maintain in Lucene writer, if specified then it tries to
-aggregate the unique data till the cache limit and flush to Lucene. It is best suitable for low
-cardinality dimensions.</li>
-<li>SPLIT_BLOCKLET: when made as true then store the data in blocklet wise in lucene , it means new
-folder will be created for each blocklet, thus, it eliminates storing blockletid in lucene and
-also it makes lucene small chunks of data.</li>
+<li>SORT_COLUMNS</li>
+<li>SORT_SCOPE</li>
+<li>TABLE_BLOCKSIZE</li>
+<li>FLAT_FOLDER</li>
+<li>LONG_STRING_COLUMNS</li>
+<li>LOCAL_DICTIONARY_ENABLE</li>
+<li>LOCAL_DICTIONARY_THRESHOLD</li>
+<li>LOCAL_DICTIONARY_EXCLUDE</li>
+<li>DICTIONARY_INCLUDE</li>
+<li>DICTIONARY_EXCLUDE</li>
+<li>INVERTED_INDEX</li>
+<li>NO_INVERTED_INDEX</li>
+<li>COLUMN_COMPRESSOR</li>
</ol>
-<h2>
-<a id="loading-data" class="anchor" href="#loading-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data</h2>
-<p>When loading data to main table, lucene index files will be generated for all the
-index_columns(String Columns) given in DMProperties which contains information about the data
-location of index_columns. These index files will be written inside a folder named with datamap name
-inside each segment folders.</p>
-<p>A system level configuration carbon.lucene.compression.mode can be added for best compression of
-lucene index files. The default value is speed, where the index writing speed will be more. If the
-value is compression, the index file size will be compressed.</p>
-<h2>
-<a id="querying-data" class="anchor" href="#querying-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Querying data</h2>
-<p>As a technique for query acceleration, Lucene indexes cannot be queried directly.
-Queries are to be made on main table. when a query with TEXT_MATCH('name:c10') or
-TEXT_MATCH_WITH_LIMIT('name:n10',10)[the second parameter represents the number of result to be
-returned, if user does not specify this value, all results will be returned without any limit] is
-fired, two jobs are fired. The first job writes the temporary files in folder created at table level
-which contains lucene's seach results and these files will be read in second job to give faster
-results. These temporary files will be cleared once the query finishes.</p>
-<p>User can verify whether a query can leverage Lucene datamap or not by executing <code>EXPLAIN</code>
-command, which will show the transformed logical plan, and thus user can check whether TEXT_MATCH()
-filter is applied on query or not.</p>
-<p><strong>Note:</strong></p>
-<ol>
+</li>
<li>
-<p>The filter columns in TEXT_MATCH or TEXT_MATCH_WITH_LIMIT must be always in lower case and
-filter condition like 'AND','OR' must be in upper case.</p>
-<p>Ex:</p>
-<pre><code>select * from datamap_test where TEXT_MATCH('name:*10 AND name:*n*')
-</code></pre>
+<p>All columns of main table at once cannot participate in mv datamap table creation</p>
</li>
<li>
-<p>Query supports only one TEXT_MATCH udf for filter condition and not multiple udfs.</p>
-<p>The following query is supported:</p>
-<pre><code>select * from datamap_test where TEXT_MATCH('name:*10 AND name:*n*')
-</code></pre>
-<p>The following query is not supported:</p>
-<pre><code>select * from datamap_test where TEXT_MATCH('name:*10) AND TEXT_MATCH(name:*n*')
-</code></pre>
+<p>TableProperties can be provided in DMProperties excluding LOCAL_DICTIONARY_INCLUDE,
+LOCAL_DICTIONARY_EXCLUDE, DICTIONARY_INCLUDE, DICTIONARY_EXCLUDE, INVERTED_INDEX,
+NO_INVERTED_INDEX, SORT_COLUMNS, LONG_STRING_COLUMNS, RANGE_COLUMN & COLUMN_META_CACHE</p>
</li>
-</ol>
-<p>Below like queries can be converted to text_match queries as following:</p>
-<pre><code>select * from datamap_test where name='n10'
-
-select * from datamap_test where name like 'n1%'
-
-select * from datamap_test where name like '%10'
+<li>
+<p>TableProperty given in DMProperties will be considered for mv creation, eventhough if same
+property is inherited from parent table, which allows user to provide different tableproperties
+for child table</p>
+</li>
+<li>
+<p>MV creation with limit or union all ctas queries is unsupported</p>
+</li>
+</ul>
+<h4>
+<a id="how-mv-tables-are-selected" class="anchor" href="#how-mv-tables-are-selected" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>How MV tables are selected</h4>
+<p>When a user query is submitted, during query planning phase, CarbonData will collect modular plan
+candidates and process the the ModularPlan based on registered summary data sets. Then,
+mv datamap table for this query will be selected among the candidates.</p>
+<p>For the main table <strong>sales</strong> and mv table <strong>agg_sales</strong> created above, following queries</p>
+<pre><code>SELECT country, sex, sum(quantity), avg(price) from sales GROUP BY country, sex
-select * from datamap_test where name like '%n%'
+SELECT sex, sum(quantity) from sales GROUP BY sex
-select * from datamap_test where name like '%10' and name not like '%n%'
+SELECT avg(price), country from sales GROUP BY country
</code></pre>
-<p>Lucene TEXT_MATCH Queries:</p>
-<pre><code>select * from datamap_test where TEXT_MATCH('name:n10')
-
-select * from datamap_test where TEXT_MATCH('name:n1*')
-
-select * from datamap_test where TEXT_MATCH('name:*10')
+<p>will be transformed by CarbonData's query planner to query against mv table
+<strong>agg_sales</strong> instead of the main table <strong>sales</strong></p>
+<p>However, for following queries</p>
+<pre><code>SELECT user_id, country, sex, sum(quantity), avg(price) from sales GROUP BY user_id, country, sex
-select * from datamap_test where TEXT_MATCH('name:*n*')
+SELECT sex, avg(quantity) from sales GROUP BY sex
-select * from datamap_test where TEXT_MATCH('name:*10 -name:*n*')
+SELECT country, max(price) from sales GROUP BY country
</code></pre>
-<p><strong>Note:</strong> For lucene queries and syntax, refer to <a href="http://www.lucenetutorial.com/lucene-query-syntax.html" target=_blank rel="nofollow">lucene-syntax</a></p>
+<p>will query against main table <strong>sales</strong> only, because it does not satisfy mv table
+selection logic.</p>
<h2>
-<a id="data-management-with-lucene-datamap" class="anchor" href="#data-management-with-lucene-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Data Management with lucene datamap</h2>
-<p>Once there is lucene datamap is created on the main table, following command on the main
-table
-is not supported:</p>
+<a id="loading-data" class="anchor" href="#loading-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data</h2>
+<h3>
+<a id="loading-data-to-non-lazy-mv-datamap" class="anchor" href="#loading-data-to-non-lazy-mv-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data to Non-Lazy MV Datamap</h3>
+<p>In case of WITHOUT DEFERRED REBUILD, for existing table with loaded data, data load to MV table will
+be triggered by the CREATE DATAMAP statement when user creates the MV table.
+For incremental loads to main table, data to datamap will be loaded once the corresponding main
+table load is completed.</p>
+<h3>
+<a id="loading-data-to-lazy-mv-datamap" class="anchor" href="#loading-data-to-lazy-mv-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data to Lazy MV Datamap</h3>
+<p>In case of WITH DEFERRED REBUILD, data load to MV table will be triggered by the <a href="./datamap-management.html#manual-refresh">Manual Refresh</a>
+command. MV datamap will be in DISABLED state in below scenarios,</p>
+<ul>
+<li>when mv datamap is created</li>
+<li>when data of main table and datamap are not in sync</li>
+</ul>
+<p>User should fire REBUILD DATAMAP command to sync all segments of main table with datamap table and
+which ENABLES the datamap for query</p>
+<h3>
+<a id="loading-data-to-multiple-mvs" class="anchor" href="#loading-data-to-multiple-mvs" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data to Multiple MV's</h3>
+<p>During load to main table, if anyone of the load to datamap table fails, then that corresponding
+datamap will be DISABLED and load to other datamaps mapped to main table will continue. User can
+fire REBUILD DATAMAP command to sync or else the subsequent table load will load the old failed
+loads along with current load and enable the disabled datamap.</p>
+<p><strong>NOTE</strong>:</p>
+<ul>
+<li>In case of InsertOverwrite/Update operation on parent table, all segments of datamap table will
+be MARKED_FOR_DELETE and reload to datamap table will happen by REBUILD DATAMAP, in case of Lazy
+mv datamap/ once InsertOverwrite/Update operation on parent table is finished, in case of
+Non-Lazy mv.</li>
+<li>In case of full scan query, Data Size and Index Size of main table and child table will not the
+same, as main table and child table has different column names.</li>
+</ul>
+<h2>
+<a id="querying-data" class="anchor" href="#querying-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Querying data</h2>
+<p>As a technique for query acceleration, MV tables cannot be queried directly.
+Queries are to be made on main table. While doing query planning, internally CarbonData will check
+associated mv datamap tables with the main table, and do query plan transformation accordingly.</p>
+<p>User can verify whether a query can leverage mv datamap table or not by executing <code>EXPLAIN</code>
+command, which will show the transformed logical plan, and thus user can check whether mv datamap
+table is selected.</p>
+<h2>
+<a id="compacting-mv-datamap" class="anchor" href="#compacting-mv-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compacting MV datamap</h2>
+<h3>
+<a id="compacting-mv-datamap-table-through-main-table-compaction" class="anchor" href="#compacting-mv-datamap-table-through-main-table-compaction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compacting MV datamap table through Main Table compaction</h3>
+<p>Running Compaction command (<code>ALTER TABLE COMPACT</code>)[COMPACTION TYPE-> MINOR/MAJOR] on main table will
+automatically compact the mv datamap tables created on the main table, once compaction on main table
+is done.</p>
+<h3>
+<a id="compacting-mv-datamap-table-through-ddl-command" class="anchor" href="#compacting-mv-datamap-table-through-ddl-command" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compacting MV datamap table through DDL command</h3>
+<p>Compaction on mv datamap can be triggered by running the following DDL command(supported only for mv).</p>
+<pre><code>ALTER DATAMAP datamap_name COMPACT 'COMPACTION_TYPE'
+</code></pre>
+<h2>
+<a id="data-management-with-mv-tables" class="anchor" href="#data-management-with-mv-tables" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Data Management with mv tables</h2>
+<p>In current implementation, data consistency needs to be maintained for both main table and mv datamap
+tables. Once there is mv datamap table created on the main table, following command on the main
+table is not supported:</p>
<ol>
-<li>Data management command: <code>UPDATE/DELETE</code>.</li>
+<li>Data management command: <code>DELETE SEGMENT</code>.</li>
<li>Schema management command: <code>ALTER TABLE DROP COLUMN</code>, <code>ALTER TABLE CHANGE DATATYPE</code>,
-<code>ALTER TABLE RENAME</code>.</li>
-</ol>
-<p><strong>Note</strong>: Adding a new column is supported, and for dropping columns and change datatype
-command, CarbonData will check whether it will impact the lucene datamap, if not, the operation
-is allowed, otherwise operation will be rejected by throwing exception.</p>
-<ol start="3">
-<li>Partition management command: <code>ALTER TABLE ADD/DROP PARTITION</code>.</li>
+<code>ALTER TABLE RENAME</code>, <code>ALTER COLUMN RENAME</code>. Note that adding a new column is supported, and for
+dropping columns and change datatype command, CarbonData will check whether it will impact the
+mv datamap table, if not, the operation is allowed, otherwise operation will be rejected by
+throwing exception.</li>
+<li>Partition management command: <code>ALTER TABLE ADD/DROP PARTITION</code>. Note that dropping a partition
+will be allowed only if partition is participating in all datamaps associated with main table.
+Drop Partition is not allowed, if any mv datamap is associated with more than one parent table.
+Drop Partition directly on datamap table is not allowed.</li>
+<li>Complex Datatype's for mv datamap is not supported.</li>
</ol>
<p>However, there is still way to support these operations on main table, in current CarbonData
release, user can do as following:</p>
<ol>
-<li>Remove the lucene datamap by <code>DROP DATAMAP</code> command.</li>
-<li>Carry out the data management operation on main table.</li>
-<li>Create the lucene datamap again by <code>CREATE DATAMAP</code> command.
+<li>Remove the mv datamap table by <code>DROP DATAMAP</code> command</li>
+<li>Carry out the data management operation on main table</li>
+<li>Create the mv datamap table again by <code>CREATE DATAMAP</code> command
Basically, user can manually trigger the operation by re-building the datamap.</li>
</ol>
<script>
$(function() {
// Show selected style on nav item
$('.b-nav__datamap').addClass('selected');
-
+
if (!$('.b-nav__datamap').parent().hasClass('nav__item__with__subs--expanded')) {
// Display datamap subnav items
$('.b-nav__datamap').parent().toggleClass('nav__item__with__subs--expanded');
diff --git a/src/main/webapp/performance-tuning.html b/src/main/webapp/performance-tuning.html
index 1fde679..2d93db9 100644
--- a/src/main/webapp/performance-tuning.html
+++ b/src/main/webapp/performance-tuning.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/preaggregate-datamap-guide.html b/src/main/webapp/preaggregate-datamap-guide.html
index b8c816d..ce95f90 100644
--- a/src/main/webapp/preaggregate-datamap-guide.html
+++ b/src/main/webapp/preaggregate-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -254,24 +263,24 @@
spark.sparkContext.setLogLevel(<span class="pl-s"><span class="pl-pds">"</span>ERROR<span class="pl-pds">"</span></span>)
<span class="pl-c"><span class="pl-c">//</span> drop table if exists previously</span>
- spark.sql(s<span class="pl-s"><span class="pl-pds">"</span>DROP TABLE IF EXISTS sales<span class="pl-pds">"</span></span>)
+ spark.sql(<span class="pl-k">s</span><span class="pl-s">"</span><span class="pl-s">DROP TABLE IF EXISTS sales</span><span class="pl-s">"</span>)
<span class="pl-c"><span class="pl-c">//</span> Create main table</span>
spark.sql(
- s<span class="pl-s"><span class="pl-pds">"""</span></span>
+ <span class="pl-k">s</span><span class="pl-s">"""</span>
<span class="pl-s"> | CREATE TABLE sales (</span>
<span class="pl-s"> | user_id string,</span>
<span class="pl-s"> | country string,</span>
<span class="pl-s"> | quantity int,</span>
<span class="pl-s"> | price bigint)</span>
<span class="pl-s"> | STORED AS carbondata</span>
-<span class="pl-s"> <span class="pl-pds">"""</span></span>.stripMargin)
+<span class="pl-s"> </span><span class="pl-s">"""</span>.stripMargin)
<span class="pl-c"><span class="pl-c">//</span> Create pre-aggregate table on the main table</span>
<span class="pl-c"><span class="pl-c">//</span> If main table already have data, following command </span>
<span class="pl-c"><span class="pl-c">//</span> will trigger one immediate load to the pre-aggregate table</span>
spark.sql(
- s<span class="pl-s"><span class="pl-pds">"""</span></span>
+ <span class="pl-k">s</span><span class="pl-s">"""</span>
<span class="pl-s"> | CREATE DATAMAP agg_sales</span>
<span class="pl-s"> | ON TABLE sales</span>
<span class="pl-s"> | USING "preaggregate"</span>
@@ -279,7 +288,7 @@
<span class="pl-s"> | SELECT country, sum(quantity), avg(price)</span>
<span class="pl-s"> | FROM sales</span>
<span class="pl-s"> | GROUP BY country</span>
-<span class="pl-s"> <span class="pl-pds">"""</span></span>.stripMargin)
+<span class="pl-s"> </span><span class="pl-s">"""</span>.stripMargin)
<span class="pl-k">import</span> <span class="pl-en">spark</span>.<span class="pl-en">implicits</span>.<span class="pl-en">_</span>
<span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">spark</span>.<span class="pl-en">sql</span>.<span class="pl-en">SaveMode</span>
@@ -302,10 +311,10 @@
.save()
spark.sql(
- s<span class="pl-s"><span class="pl-pds">"""</span></span>
+ <span class="pl-k">s</span><span class="pl-s">"""</span>
<span class="pl-s"> |SELECT country, sum(quantity), avg(price)</span>
<span class="pl-s"> | from sales GROUP BY country</span>
-<span class="pl-s"> <span class="pl-pds">"""</span></span>.stripMargin).show
+<span class="pl-s"> </span><span class="pl-s">"""</span>.stripMargin).show
spark.stop</pre></div>
<h4>
@@ -366,6 +375,11 @@ AS
FROM sales
GROUP BY country, sex
</code></pre>
+<p><strong>NOTE</strong>:</p>
+<ul>
+<li>Preaggregate datamap is deprecated and it is replaced by MV datamap.
+Please refer <a href="./mv-datamap-guide.html">CarbonData MV DataMap</a> for more info.</li>
+</ul>
<h4>
<a id="functions-supported-in-pre-aggregate-table" class="anchor" href="#functions-supported-in-pre-aggregate-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Functions supported in pre-aggregate table</h4>
<table>
diff --git a/src/main/webapp/presto-guide.html b/src/main/webapp/presto-guide.html
index b104bdb..af899f8 100644
--- a/src/main/webapp/presto-guide.html
+++ b/src/main/webapp/presto-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/quick-start-guide.html b/src/main/webapp/quick-start-guide.html
index 61cedd3..d3c788c 100644
--- a/src/main/webapp/quick-start-guide.html
+++ b/src/main/webapp/quick-start-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/release-guide.html b/src/main/webapp/release-guide.html
index bbb02b5..e034a28 100644
--- a/src/main/webapp/release-guide.html
+++ b/src/main/webapp/release-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/s3-guide.html b/src/main/webapp/s3-guide.html
index 41fd0ae..3c3c597 100644
--- a/src/main/webapp/s3-guide.html
+++ b/src/main/webapp/s3-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/sdk-guide.html b/src/main/webapp/sdk-guide.html
index 23dd148..3e7f999 100644
--- a/src/main/webapp/sdk-guide.html
+++ b/src/main/webapp/sdk-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -421,6 +430,11 @@ These SDK writer output contains just carbondata and carbonindex files. No metad
<td>DataTypes.STRING</td>
</tr>
<tr>
+<td>BINARY</td>
+<td>-</td>
+<td>DataTypes.BINARY</td>
+</tr>
+<tr>
<td>FLOAT</td>
<td>FLOAT</td>
<td>DataTypes.FLOAT</td>
@@ -511,7 +525,7 @@ c. Timestamp (microsecond precision)
The timestamp-micros logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one microsecond.
A timestamp-micros logical type annotates an Avro long, where the long stores the number of microseconds from the unix epoch, 1 January 1970 00:00:00.000000 UTC.
d. Decimal
-The decimal logical type represents an arbitrary-precision signed decimal number of the form unscaled � 10-scale.
+The decimal logical type represents an arbitrary-precision signed decimal number of the form <em>unscaled × 10<sup>-scale</sup></em>.
A decimal logical type annotates Avro bytes or fixed types. The byte array must contain the two's-complement representation of the unscaled integer value in big-endian byte order. The scale is fixed, and is specified using an attribute.
e. Time (millisecond precision)
The time-millis logical type represents a time of day, with no reference to a particular calendar, time zone or date, with a precision of one millisecond.
@@ -655,6 +669,7 @@ public CarbonWriterBuilder withLoadOptions(Map<String, String> options);
* default value is null.
* l. inverted_index -- comma separated string columns for which inverted index needs to be
* generated
+ * m. table_page_size_inmb -- [1-1755] MB.
*
* @return updated CarbonWriterBuilder
*/
@@ -850,6 +865,8 @@ while (reader.hasNext()) {
reader.close();
</code></pre>
<p>Find example code at <a href="https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java" target=_blank>CarbonReaderExample</a> in the CarbonData repo.</p>
+<p>SDK reader also supports reading carbondata files and filling it to apache arrow vectors.
+Find example code at <a href="https://github.com/apache/carbondata/blob/master/store/sdk/src/test/java/org/apache/carbondata/sdk/file/ArrowCarbonReaderTest.java" target=_blank>ArrowCarbonReaderTest</a> in the CarbonData repo.</p>
<h2>
<a id="api-list-1" class="anchor" href="#api-list-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>API List</h2>
<h3>
@@ -911,6 +928,72 @@ public Object[] readNextBatchRow();
public void close();
</code></pre>
<h3>
+<a id="class-orgapachecarbondatasdkfilearrowcarbonreader" class="anchor" href="#class-orgapachecarbondatasdkfilearrowcarbonreader" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.ArrowCarbonReader</h3>
+<pre><code>/**
+ * Carbon reader will fill the arrow vector after reading the carbondata files.
+ * This arrow byte[] can be used to create arrow table and used for in memory analytics
+ * Note: create a reader at blocklet level, so that arrow byte[] will not exceed INT_MAX
+ *
+ * @param carbonSchema org.apache.carbondata.sdk.file.Schema
+ * @return Serialized byte array
+ * @throws Exception
+ */
+public byte[] readArrowBatch(Schema carbonSchema) throws Exception;
+</code></pre>
+<pre><code>/**
+ * Carbon reader will fill the arrow vector after reading the carbondata files.
+ * This arrow byte[] can be used to create arrow table and used for in memory analytics
+ * Note: create a reader at blocklet level, so that arrow byte[] will not exceed INT_MAX
+ * User need to close the VectorSchemaRoot after usage by calling VectorSchemaRoot.close()
+ *
+ * @param carbonSchema org.apache.carbondata.sdk.file.Schema
+ * @return Arrow VectorSchemaRoot
+ * @throws Exception
+ */
+public VectorSchemaRoot readArrowVectors(Schema carbonSchema) throws Exception;
+</code></pre>
+<pre><code>/**
+ * Carbon reader will fill the arrow vector after reading carbondata files.
+ * Here unsafe memory address will be returned instead of byte[],
+ * so that this address can be sent across java to python or c modules and
+ * can directly read the content from this unsafe memory
+ * Note:Create a carbon reader at blocklet level using CarbonReader.buildWithSplits(split) method,
+ * so that arrow byte[] will not exceed INT_MAX.
+ *
+ * @param carbonSchema org.apache.carbondata.sdk.file.Schema
+ * @return address of the unsafe memory where arrow buffer is stored
+ * @throws Exception
+ */
+public long readArrowBatchAddress(Schema carbonSchema) throws Exception;
+</code></pre>
+<pre><code>/**
+ * Free the unsafe memory allocated , if unsafe arrow batch is used.
+ *
+ * @param address address of the unsafe memory where arrow bufferer is stored
+ */
+public void freeArrowBatchMemory(long address)
+</code></pre>
+<h3>
+<a id="class-orgapachecarbondatasdkfilearrowarrowconverter" class="anchor" href="#class-orgapachecarbondatasdkfilearrowarrowconverter" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.arrow.ArrowConverter</h3>
+<pre><code>/**
+ * To get the arrow vectors directly after filling from carbondata
+ *
+ * @return Arrow VectorSchemaRoot. which contains array of arrow vectors.
+ */
+public VectorSchemaRoot getArrowVectors() throws IOException;
+</code></pre>
+<pre><code>/**
+ * Utility API to convert back the arrow byte[] to arrow ArrowRecordBatch.
+ * User need to close the ArrowRecordBatch after usage by calling ArrowRecordBatch.close()
+ *
+ * @param batchBytes input byte array
+ * @param bufferAllocator arrow buffer allocator
+ * @return ArrowRecordBatch
+ * @throws IOException
+ */
+public static ArrowRecordBatch byteArrayToArrowBatch(byte[] batchBytes, BufferAllocator bufferAllocator) throws IOException;
+</code></pre>
+<h3>
<a id="class-orgapachecarbondatasdkfilecarbonreaderbuilder" class="anchor" href="#class-orgapachecarbondatasdkfilecarbonreaderbuilder" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.CarbonReaderBuilder</h3>
<pre><code>/**
* Construct a CarbonReaderBuilder with table path and table name
diff --git a/src/main/webapp/security.html b/src/main/webapp/security.html
index b9ba692..0cbb03f 100644
--- a/src/main/webapp/security.html
+++ b/src/main/webapp/security.html
@@ -45,6 +45,12 @@
aria-expanded="false">Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
diff --git a/src/main/webapp/segment-management-on-carbondata.html b/src/main/webapp/segment-management-on-carbondata.html
index b80b22c..bb0d793 100644
--- a/src/main/webapp/segment-management-on-carbondata.html
+++ b/src/main/webapp/segment-management-on-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/streaming-guide.html b/src/main/webapp/streaming-guide.html
index 7f3b414..55f9f76 100644
--- a/src/main/webapp/streaming-guide.html
+++ b/src/main/webapp/streaming-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -287,16 +296,16 @@
spark.sparkContext.setLogLevel(<span class="pl-s"><span class="pl-pds">"</span>ERROR<span class="pl-pds">"</span></span>)
<span class="pl-c"><span class="pl-c">//</span> drop table if exists previously</span>
- spark.sql(s<span class="pl-s"><span class="pl-pds">"</span>DROP TABLE IF EXISTS carbon_table<span class="pl-pds">"</span></span>)
+ spark.sql(<span class="pl-k">s</span><span class="pl-s">"</span><span class="pl-s">DROP TABLE IF EXISTS carbon_table</span><span class="pl-s">"</span>)
<span class="pl-c"><span class="pl-c">//</span> Create target carbon table and populate with initial data</span>
spark.sql(
- s<span class="pl-s"><span class="pl-pds">"""</span></span>
+ <span class="pl-k">s</span><span class="pl-s">"""</span>
<span class="pl-s"> | CREATE TABLE carbon_table (</span>
<span class="pl-s"> | col1 INT,</span>
<span class="pl-s"> | col2 STRING</span>
<span class="pl-s"> | )</span>
<span class="pl-s"> | STORED AS carbondata</span>
-<span class="pl-s"> | TBLPROPERTIES('streaming'='true')<span class="pl-pds">"""</span></span>.stripMargin)
+<span class="pl-s"> | TBLPROPERTIES('streaming'='true')</span><span class="pl-s">"""</span>.stripMargin)
<span class="pl-k">val</span> <span class="pl-smi">carbonTable</span> <span class="pl-k">=</span> <span class="pl-en">CarbonEnv</span>.getCarbonTable(<span class="pl-en">Some</span>(<span class="pl-s"><span class="pl-pds">"</span>default<span class="pl-pds">"</span></span>), <span class="pl-s"><span class="pl-pds">"</span>carbon_table<span class="pl-pds">"</span></span>)(spark)
<span class="pl-k">val</span> <span class="pl-smi">tablePath</span> <span class="pl-k">=</span> carbonTable.getTablePath
diff --git a/src/main/webapp/supported-data-types-in-carbondata.html b/src/main/webapp/supported-data-types-in-carbondata.html
index c00f28d..3c4a32c 100644
--- a/src/main/webapp/supported-data-types-in-carbondata.html
+++ b/src/main/webapp/supported-data-types-in-carbondata.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
@@ -269,6 +278,7 @@ Please refer to TBLProperties in <a href="./ddl-of-carbondata.html#create-table"
<p>Other Types</p>
<ul>
<li>BOOLEAN</li>
+<li>BINARY</li>
</ul>
</li>
</ul>
diff --git a/src/main/webapp/timeseries-datamap-guide.html b/src/main/webapp/timeseries-datamap-guide.html
index b3770ee..a0bf828 100644
--- a/src/main/webapp/timeseries-datamap-guide.html
+++ b/src/main/webapp/timeseries-datamap-guide.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/usecases.html b/src/main/webapp/usecases.html
index fa2c0ea..e22945f 100644
--- a/src/main/webapp/usecases.html
+++ b/src/main/webapp/usecases.html
@@ -52,6 +52,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
@@ -180,6 +186,7 @@
<a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ <a class="nav__item nav__sub__item" href="./mv-datamap-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
@@ -190,6 +197,7 @@
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
@@ -205,6 +213,7 @@
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
diff --git a/src/main/webapp/videogallery.html b/src/main/webapp/videogallery.html
index 614d8c6..6eb4503 100644
--- a/src/main/webapp/videogallery.html
+++ b/src/main/webapp/videogallery.html
@@ -49,6 +49,12 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
+ target="_blank">Apache CarbonData 1.6.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
+ target="_blank">Apache CarbonData 1.5.4</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
diff --git a/src/site/markdown/carbon-as-spark-datasource-guide.md b/src/site/markdown/carbon-as-spark-datasource-guide.md
index bc56a54..b61bf43 100644
--- a/src/site/markdown/carbon-as-spark-datasource-guide.md
+++ b/src/site/markdown/carbon-as-spark-datasource-guide.md
@@ -19,7 +19,7 @@
The CarbonData fileformat is now integrated as Spark datasource for read and write operation without using CarbonSession. This is useful for users who wants to use carbondata as spark's data source.
-**Note:** You can only apply the functions/features supported by spark datasource APIs, functionalities supported would be similar to Parquet. The carbon session features are not supported.
+**Note:** You can only apply the functions/features supported by spark datasource APIs, functionalities supported would be similar to Parquet. The carbon session features are not supported. The result is displayed as byte array format when select query on binary column in spark-sql.
# Create Table with DDL
@@ -42,18 +42,20 @@ Now you can create Carbon table using Spark's datasource DDL syntax.
| Property | Default Value | Description |
|-----------|--------------|------------|
-| table_blocksize | 1024 | Size of blocks to write onto hdfs. For more details, see [Table Block Size Configuration](./ddl-of-carbondata.md#table-block-size-configuration). |
+| table_blocksize | 1024 | Size of blocks to write onto hdfs. For more details, see [Table Block Size Configuration](./ddl-of-carbondata.md#table-block-size-configuration). |
| table_blocklet_size | 64 | Size of blocklet to write. |
-| local_dictionary_threshold | 10000 | Cardinality upto which the local dictionary can be generated. For more details, see [Local Dictionary Configuration](./ddl-of-carbondata.md#local-dictionary-configuration). |
-| local_dictionary_enable | false | Enable local dictionary generation. For more details, see [Local Dictionary Configuration](./ddl-of-carbondata.md#local-dictionary-configuration). |
-| sort_columns | all dimensions are sorted | Columns to include in sort and its order of sort. For more details, see [Sort Columns Configuration](./ddl-of-carbondata.md#sort-columns-configuration). |
-| sort_scope | local_sort | Sort scope of the load.Options include no sort, local sort, batch sort, and global sort. For more details, see [Sort Scope Configuration](./ddl-of-carbondata.md#sort-scope-configuration). |
-| long_string_columns | null | Comma separated string/char/varchar columns which are more than 32k length. For more details, see [String longer than 32000 characters](./ddl-of-carbondata.md#string-longer-than-32000-characters). |
-
+| table_page_size_inmb | 0 | Size of each page in carbon table, if page size crosses this value before 32000 rows, page will be cut to that many rows. Helps in keep page size to fit cache size |
+| local_dictionary_threshold | 10000 | Cardinality upto which the local dictionary can be generated. For more details, see [Local Dictionary Configuration](./ddl-of-carbondata.md#local-dictionary-configuration). |
+| local_dictionary_enable | false | Enable local dictionary generation. For more details, see [Local Dictionary Configuration](./ddl-of-carbondata.md#local-dictionary-configuration). |
+| sort_columns | all dimensions are sorted | Columns to include in sort and its order of sort. For more details, see [Sort Columns Configuration](./ddl-of-carbondata.md#sort-columns-configuration). |
+| sort_scope | local_sort | Sort scope of the load.Options include no sort, local sort, batch sort, and global sort. For more details, see [Sort Scope Configuration](./ddl-of-carbondata.md#sort-scope-configuration). |
+| long_string_columns | null | Comma separated string/char/varchar columns which are more than 32k length. For more details, see [String longer than 32000 characters](./ddl-of-carbondata.md#string-longer-than-32000-characters). |
+
+ **NOTE:** please set long_string_columns for varchar column.
## Example
```
- CREATE TABLE CARBON_TABLE (NAME STRING) USING CARBON OPTIONS('table_block_size'='256')
+ CREATE TABLE CARBON_TABLE (NAME STRING) USING CARBON OPTIONS('table_block_size'='256')
```
# Using DataFrame
diff --git a/src/site/markdown/configuration-parameters.md b/src/site/markdown/configuration-parameters.md
index 7b31413..da226ec 100644
--- a/src/site/markdown/configuration-parameters.md
+++ b/src/site/markdown/configuration-parameters.md
@@ -48,6 +48,7 @@ This section provides the details of all the configurations required for the Car
| carbon.invisible.segments.preserve.count | 200 | CarbonData maintains each data load entry in tablestatus file. The entries from this file are not deleted for those segments that are compacted or dropped, but are made invisible. If the number of data loads are very high, the size and number of entries in tablestatus file can become too many causing unnecessary reading of all data. This configuration specifies the number of segment entries to be maintained afte they are compacted or dro [...]
| carbon.lock.retries | 3 | CarbonData ensures consistency of operations by blocking certain operations from running in parallel. In order to block the operations from running in parallel, lock is obtained on the table. This configuration specifies the maximum number of retries to obtain the lock for any operations other than load. **NOTE:** Data manupulation operations like Compaction,UPDATE,DELETE or LOADING,UPDATE,DELETE are not allowed to run in parallel. How ever data loading can h [...]
| carbon.lock.retry.timeout.sec | 5 | Specifies the interval between the retries to obtain the lock for any operation other than load. **NOTE:** Refer to ***carbon.lock.retries*** for understanding why CarbonData uses locks for operations. |
+| carbon.fs.custom.file.provider | None | To support FileTypeInterface for configuring custom CarbonFile implementation to work with custom FileSystem. |
## Data Loading Configuration
@@ -93,6 +94,8 @@ This section provides the details of all the configurations required for the Car
| carbon.options.serialization.null.format | \N | Based on the business scenarios, some columns might need to be loaded with null values. As null value cannot be written in csv files, some special characters might be adopted to specify null values. This configuration can be used to specify the null values format in the data being loaded. |
| carbon.column.compressor | snappy | CarbonData will compress the column values using the compressor specified by this configuration. Currently CarbonData supports 'snappy', 'zstd' and 'gzip' compressors. |
| carbon.minmax.allowed.byte.count | 200 | CarbonData will write the min max values for string/varchar types column using the byte count specified by this configuration. Max value is 1000 bytes(500 characters) and Min value is 10 bytes(5 characters). **NOTE:** This property is useful for reducing the store size thereby improving the query performance but can lead to query degradation if value is not configured properly. | |
+| carbon.merge.index.failure.throw.exception | true | It is used to configure whether or not merge index failure should result in data load failure also. |
+| carbon.binary.decoder | None | Support configurable decode for loading. Two decoders supported: base64 and hex |
## Compaction Configuration
@@ -112,6 +115,7 @@ This section provides the details of all the configurations required for the Car
| carbon.concurrent.compaction | true | Compaction of different tables can be executed concurrently. This configuration determines whether to compact all qualifying tables in parallel or not. **NOTE: **Compacting concurrently is a resource demanding operation and needs more resources there by affecting the query performance also. This configuration is **deprecated** and might be removed in future releases. |
| carbon.compaction.prefetch.enable | false | Compaction operation is similar to Query + data load where in data from qualifying segments are queried and data loading performed to generate a new single segment. This configuration determines whether to query ahead data from segments and feed it for data loading. **NOTE: **This configuration is disabled by default as it needs extra resources for querying extra data. Based on the memory availability on the cluster, user can enable it to imp [...]
| carbon.merge.index.in.segment | true | Each CarbonData file has a companion CarbonIndex file which maintains the metadata about the data. These CarbonIndex files are read and loaded into driver and is used subsequently for pruning of data during queries. These CarbonIndex files are very small in size(few KB) and are many. Reading many small files from HDFS is not efficient and leads to slow IO performance. Hence these CarbonIndex files belonging to a segment can be combined into a sin [...]
+| carbon.enable.range.compaction | true | To configure Ranges-based Compaction to be used or not for RANGE_COLUMN. If true after compaction also the data would be present in ranges. |
## Query Configuration
@@ -138,6 +142,7 @@ This section provides the details of all the configurations required for the Car
| carbon.max.driver.threads.for.block.pruning | 4 | Number of threads used for driver pruning when the carbon files are more than 100k Maximum memory. This configuration can used to set number of threads between 1 to 4. |
| carbon.heap.memory.pooling.threshold.bytes | 1048576 | CarbonData supports unsafe operations of Java to avoid GC overhead for certain operations. Using unsafe, memory can be allocated on Java Heap or off heap. This configuration controls the allocation mechanism on Java HEAP. If the heap memory allocations of the given size is greater or equal than this value,it should go through the pooling mechanism. But if set this size to -1, it should not go through the pooling mechanism. Default [...]
| carbon.push.rowfilters.for.vector | false | When enabled complete row filters will be handled by carbon in case of vector. If it is disabled then only page level pruning will be done by carbon and row level filtering will be done by spark for vector. And also there are scan optimizations in carbon to avoid multiple data copies when this parameter is set to false. There is no change in flow for non-vector based queries. |
+| carbon.query.prefetch.enable | true | By default this property is true, so prefetch is used in query to read next blocklet asynchronously in other thread while processing current blocklet in main thread. This can help to reduce CPU idle time. Setting this property false will disable this prefetch feature in query. |
## Data Mutation Configuration
| Parameter | Default Value | Description |
diff --git a/src/site/markdown/datamap-management.md b/src/site/markdown/datamap-management.md
index 087c70a..199cd14 100644
--- a/src/site/markdown/datamap-management.md
+++ b/src/site/markdown/datamap-management.md
@@ -49,7 +49,7 @@ Currently, there are 5 DataMap implementations in CarbonData.
| ---------------- | ---------------------------------------- | ---------------------------------------- | ---------------- |
| preaggregate | single table pre-aggregate table | No DMPROPERTY is required | Automatic |
| timeseries | time dimension rollup table | event_time, xx_granularity, please refer to [Timeseries DataMap](./timeseries-datamap-guide.md) | Automatic |
-| mv | multi-table pre-aggregate table | No DMPROPERTY is required | Manual |
+| mv | multi-table pre-aggregate table | No DMPROPERTY is required | Manual/Automatic |
| lucene | lucene indexing for text column | index_columns to specifying the index columns | Automatic |
| bloomfilter | bloom filter for high cardinality column, geospatial column | index_columns to specifying the index columns | Automatic |
@@ -60,9 +60,6 @@ There are two kinds of management semantic for DataMap.
1. Automatic Refresh: Create datamap without `WITH DEFERRED REBUILD` in the statement, which is by default.
2. Manual Refresh: Create datamap with `WITH DEFERRED REBUILD` in the statement
-**CAUTION:**
-If user create MV datamap without specifying `WITH DEFERRED REBUILD`, carbondata will give a warning and treat the datamap as deferred rebuild.
-
### Automatic Refresh
When user creates a datamap on the main table without using `WITH DEFERRED REBUILD` syntax, the datamap will be managed by system automatically.
@@ -142,6 +139,9 @@ There is a SHOW DATAMAPS command, when this is issued, system will read all data
- DataMapProviderName like mv, preaggreagte, timeseries, etc
- Associated Table
- DataMap Properties
+- DataMap status (ENABLED/DISABLED)
+- Sync Status - which displays Last segment Id of main table synced with datamap table and its load
+ end time (Applicable only for mv datamap)
### Compaction on DataMap
diff --git a/src/site/markdown/ddl-of-carbondata.md b/src/site/markdown/ddl-of-carbondata.md
index 07a2670..7ab0e5f 100644
--- a/src/site/markdown/ddl-of-carbondata.md
+++ b/src/site/markdown/ddl-of-carbondata.md
@@ -94,6 +94,7 @@ CarbonData DDL statements are documented here,which includes:
| [SORT_SCOPE](#sort-scope-configuration) | Sort scope of the load.Options include no sort, local sort ,batch sort and global sort |
| [TABLE_BLOCKSIZE](#table-block-size-configuration) | Size of blocks to write onto hdfs |
| [TABLE_BLOCKLET_SIZE](#table-blocklet-size-configuration) | Size of blocklet to write in the file |
+| [TABLE_PAGE_SIZE_INMB](#table-page-size-configuration) | Size of page in MB; if page size crosses this value before 32000 rows, page will be cut to this many rows and remaining rows are processed in the subsequent pages. This helps in keeping page size to fit in cpu cache size|
| [MAJOR_COMPACTION_SIZE](#table-compaction-configuration) | Size upto which the segments can be combined into one |
| [AUTO_LOAD_MERGE](#table-compaction-configuration) | Whether to auto compact the segments |
| [COMPACTION_LEVEL_THRESHOLD](#table-compaction-configuration) | Number of segments to compact into one segment |
@@ -125,7 +126,7 @@ CarbonData DDL statements are documented here,which includes:
```
**NOTE**:
- * Dictionary Include/Exclude for complex child columns is not supported.
+ * Dictionary Include/Exclude for complex child columns is not supported. Dictionary Include doesn't support binary data type.
* Dictionary is global. Except global dictionary, there are local dictionary and non-dictionary in CarbonData.
- ##### Local Dictionary Configuration
@@ -156,6 +157,7 @@ CarbonData DDL statements are documented here,which includes:
* BOOLEAN
* FLOAT
* BYTE
+ * Binary
* In case of multi-level complex dataType columns, primitive string/varchar/char columns are considered for local dictionary generation.
System Level Properties for Local Dictionary:
@@ -163,7 +165,7 @@ CarbonData DDL statements are documented here,which includes:
| Properties | Default value | Description |
| ---------- | ------------- | ----------- |
- | carbon.local.dictionary.enable | false | By default, Local Dictionary will be disabled for the carbondata table. |
+ | carbon.local.dictionary.enable | true | By default, Local Dictionary will be enabled for the carbondata table. |
| carbon.local.dictionary.decoder.fallback | true | Page Level data will not be maintained for the blocklet. During fallback, actual data will be retrieved from the encoded page data using local dictionary. **NOTE:** Memory footprint decreases significantly as compared to when this property is set to false |
Local Dictionary can be configured using the following properties during create table command:
@@ -223,7 +225,7 @@ CarbonData DDL statements are documented here,which includes:
- ##### Sort Columns Configuration
This property is for users to specify which columns belong to the MDK(Multi-Dimensions-Key) index.
- * If users don't specify "SORT_COLUMN" property, by default no columns are sorted
+ * If users don't specify "SORT_COLUMNS" property, by default no columns are sorted
* If this property is specified but with empty argument, then the table will be loaded without sort.
* This supports only string, date, timestamp, short, int, long, byte and boolean data types.
Suggested use cases : Only build MDK index for required columns,it might help to improve the data loading performance.
@@ -232,7 +234,7 @@ CarbonData DDL statements are documented here,which includes:
TBLPROPERTIES ('SORT_COLUMNS'='column1, column3')
```
- **NOTE**: Sort_Columns for Complex datatype columns is not supported.
+ **NOTE**: Sort_Columns for Complex datatype columns and binary data type is not supported.
- ##### Sort Scope Configuration
@@ -283,6 +285,23 @@ CarbonData DDL statements are documented here,which includes:
TBLPROPERTIES ('TABLE_BLOCKLET_SIZE'='8')
```
+ - ##### Table page Size Configuration
+
+ This property is for setting page size in the carbondata file
+ and supports a range of 1 MB to 1755 MB.
+ If page size crosses this value before 32000 rows, page will be cut to that many rows.
+ Helps in keeping page size to fit cpu cache size.
+
+ This property can be configured if the table has string, varchar, binary or complex datatype columns.
+ Because for these columns 32000 rows in one page may exceed 1755 MB and snappy compression will fail in that scenario.
+ Also if page size is huge, page cannot be fit in CPU cache.
+ So, configuring smaller values of this property (say 1 MB) can result in better use of CPU cache for pages.
+
+ Example usage:
+ ```
+ TBLPROPERTIES ('TABLE_PAGE_SIZE_INMB'='5')
+ ```
+
- ##### Table Compaction Configuration
These properties are table level compaction configurations, if not specified, system level configurations in carbon.properties will be used.
@@ -313,7 +332,7 @@ CarbonData DDL statements are documented here,which includes:
- ##### Caching Min/Max Value for Required Columns
- By default, CarbonData caches min and max values of all the columns in schema. As the load increases, the memory required to hold the min and max values increases considerably. This feature enables you to configure min and max values only for the required columns, resulting in optimized memory usage.
+ By default, CarbonData caches min and max values of all the columns in schema. As the load increases, the memory required to hold the min and max values increases considerably. This feature enables you to configure min and max values only for the required columns, resulting in optimized memory usage. This feature doesn't support binary data type.
Following are the valid values for COLUMN_META_CACHE:
* If you want no column min/max values to be cached in the driver.
@@ -501,6 +520,7 @@ CarbonData DDL statements are documented here,which includes:
- ##### Range Column
This property is used to specify a column to partition the input data by range.
Only one column can be configured. During data loading, you can use "global_sort_partitions" or "scale_factor" to avoid generating small files.
+ This feature doesn't support binary data type.
```
TBLPROPERTIES('RANGE_COLUMN'='col1')
@@ -780,6 +800,28 @@ Users can specify which columns to include and exclude for local dictionary gene
ALTER TABLE tablename UNSET TBLPROPERTIES('SORT_SCOPE')
```
+ - ##### SORT COLUMNS
+ Example to SET SORT COLUMNS:
+ ```
+ ALTER TABLE tablename SET TBLPROPERTIES('SORT_COLUMNS'='column1')
+ ```
+ After this operation, the new loading will use the new SORT_COLUMNS. The user can adjust
+ the SORT_COLUMNS according to the query, but it will not impact the old data directly. So
+ it will not impact the query performance of the old data segments which are not sorted by
+ new SORT_COLUMNS.
+
+ UNSET is not supported, but it can set SORT_COLUMNS to empty string instead of using UNSET.
+ NOTE: When SORT_SCOPE is not NO_SORT, then setting SORT_COLUMNS to empty string is not valid.
+ ```
+ ALTER TABLE tablename SET TBLPROPERTIES('SORT_COLUMNS'='')
+ ```
+
+ **NOTE:**
+ * The future version will enhance "custom" compaction to sort the old segment one by one.
+ * The streaming table is not supported for SORT_COLUMNS modification.
+ * If the inverted index columns are removed from the new SORT_COLUMNS, they will not
+ create the inverted index. But the old configuration of INVERTED_INDEX will be kept.
+
### DROP TABLE
This command is used to delete an existing table.
@@ -877,7 +919,7 @@ Users can specify which columns to include and exclude for local dictionary gene
PARTITIONED BY (productCategory STRING, productBatch STRING)
STORED AS carbondata
```
- **NOTE:** Hive partition is not supported on complex datatype columns.
+ **NOTE:** Hive partition is not supported on complex data type columns.
#### Show Partitions
diff --git a/src/site/markdown/dml-of-carbondata.md b/src/site/markdown/dml-of-carbondata.md
index 6ec0520..84c629c 100644
--- a/src/site/markdown/dml-of-carbondata.md
+++ b/src/site/markdown/dml-of-carbondata.md
@@ -70,6 +70,7 @@ CarbonData DML statements are documented here,which includes:
| [IS_EMPTY_DATA_BAD_RECORD](#bad-records-handling) | Whether empty data of a column to be considered as bad record or not |
| [GLOBAL_SORT_PARTITIONS](#global_sort_partitions) | Number of partition to use for shuffling of data during sorting |
| [SCALE_FACTOR](#scale_factor) | Control the partition size for RANGE_COLUMN feature |
+| [CARBON_OPTIONS_BINARY_DECODER] | Support configurable decode for loading from csv |
-
You can use the following options to load data:
@@ -281,6 +282,8 @@ CarbonData DML statements are documented here,which includes:
If the SORT_SCOPE is defined as GLOBAL_SORT, then user can specify the number of partitions to use while shuffling data for sort using GLOBAL_SORT_PARTITIONS. If it is not configured, or configured less than 1, then it uses the number of map task as reduce task. It is recommended that each reduce task deal with 512MB-1GB data.
For RANGE_COLUMN, GLOBAL_SORT_PARTITIONS is used to specify the number of range partitions also.
+ GLOBAL_SORT_PARTITIONS should be specified optimally during RANGE_COLUMN LOAD because if a higher number is configured then the load time may be less but it will result in creation of more files which would degrade the query and compaction performance.
+ Conversely, if less partitions are configured then the load performance may degrade due to less use of parallelism but the query and compaction will become faster. Hence the user may choose optimal number depending on the use case.
```
OPTIONS('GLOBAL_SORT_PARTITIONS'='2')
```
@@ -305,6 +308,11 @@ CarbonData DML statements are documented here,which includes:
* If both GLOBAL_SORT_PARTITIONS and SCALE_FACTOR are used at the same time, only GLOBAL_SORT_PARTITIONS is valid.
* The compaction on RANGE_COLUMN will use LOCAL_SORT by default.
+ - ##### CARBON_ENABLE_RANGE_COMPACTION
+
+ To configure Ranges-based Compaction to be used or not for RANGE_COLUMN.
+ The default value is 'true'.
+
### INSERT DATA INTO CARBONDATA TABLE
This command inserts data into a CarbonData table, it is defined as a combination of two queries Insert and Select query respectively.
diff --git a/src/site/markdown/hive-guide.md b/src/site/markdown/hive-guide.md
index 0e52f1b..e839b9b 100644
--- a/src/site/markdown/hive-guide.md
+++ b/src/site/markdown/hive-guide.md
@@ -18,9 +18,6 @@
# Quick Start
This tutorial provides a quick introduction to using current integration/hive module.
-## Build (In 1.2.0, hive integration only support spark2.1 and hadoop2.7.2)
-mvn -DskipTests -Pspark-2.1 -Phadoop-2.7.2 clean package
-
## Prepare CarbonData in Spark
* Create a sample.csv file using the following commands. The CSV file is required for loading data into CarbonData.
@@ -83,20 +80,42 @@ copy snappy-java-xxx.jar from "./<SPARK_HOME>/jars/" to "./Library/Java/Extensio
export HADOOP_OPTS="-Dorg.xerial.snappy.lib.path=/Library/Java/Extensions -Dorg.xerial.snappy.lib.name=libsnappyjava.jnilib -Dorg.xerial.snappy.tempdir=/Users/apple/DEMO/tmp"
```
-### Start hive client
+### Carbon Jars to be placed
+```
+hive/lib/ (for hive server)
+yarn/lib/ (for MapReduce)
+
+Carbon Jars to be copied to the above paths.
+```
+
+### Start hive beeline to query
```
-$HIVE_HOME/bin/hive
+$HIVE_HOME/bin/beeline
```
-### Query data from hive table
+### Query data from hive
+
+ - This is to read the carbon table through Hive. It is the integration of the carbon with Hive.
```
set hive.mapred.supports.subdirectories=true;
-set mapreduce.input.fileinputformat.input.dir.recursive=true;
+set mapreduce.dir.recursive=true;
+These properties helps to recursively traverse through the directories to read the carbon folder structure.
+```
-select * from hive_carbon;
-select count(*) from hive_carbon;
-select * from hive_carbon order by id;
+### Example
```
+ - In case if the carbon table is not set with the SERDE and the INPUTFORMAT/OUTPUTFORMAT, user can create a new hive managed table like below with the required details for the hive to read.
+create table hive_carbon_1(id int, name string, scale decimal, country string, salary double) ROW FORMAT SERDE 'org.apache.carbondata.hive.CarbonHiveSerDe' WITH SERDEPROPERTIES ('mapreduce.input.carboninputformat.databaseName'='default', 'mapreduce.input.carboninputformat.tableName'='HIVE_CARBON_EXAMPLE') STORED AS INPUTFORMAT 'org.apache.carbondata.hive.MapredCarbonInputFormat' OUTPUTFORMAT 'org.apache.carbondata.hive.MapredCarbonOutputFormat' LOCATION 'location_to_the_carbon_table';
+
+ - Query the table
+select * from hive_carbon_1;
+select count(*) from hive_carbon_1;
+select * from hive_carbon_1 order by id;
+```
+
+### Note
+ - Partition table support is not handled
+ - Map data type is not supported
diff --git a/src/site/markdown/index-server.md b/src/site/markdown/index-server.md
new file mode 100644
index 0000000..9253f2a
--- /dev/null
+++ b/src/site/markdown/index-server.md
@@ -0,0 +1,227 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to you under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Distributed Index Server
+
+## Background
+
+Carbon currently prunes and caches all block/blocklet datamap index information into the driver for
+normal table, for Bloom/Index datamaps the JDBC driver will launch a job to prune and cache the
+datamaps in executors.
+
+This causes the driver to become a bottleneck in the following ways:
+1. If the cache size becomes huge(70-80% of the driver memory) then there can be excessive GC in
+the driver which can slow down the query and the driver may even go OutOfMemory.
+2. LRU has to evict a lot of elements from the cache to accommodate the new objects which would
+in turn slow down the queries.
+3. For bloom there is no guarantee that the next query goes to the same executor to reuse the cache
+and hence cache could be duplicated in multiple executors.
+4. Multiple JDBC drivers need to maintain their own copy of the cache.
+
+Distributed Index Cache Server aims to solve the above mentioned problems.
+
+## Distribution
+When enabled, any query on a carbon table will be routed to the index server service in form of
+a request. The request will consist of the table name, segments, filter expression and other
+information used for pruning.
+
+In IndexServer service a pruning RDD is fired which will take care of the pruning for that
+request. This RDD will be creating tasks based on the number of segments that are applicable for
+pruning. It can happen that the user has specified segments to access for that table, so only the
+specified segments would be applicable for pruning. Refer: [query-data-with-specified-segments](https://github.com/apache/carbondata/blob/6e50c1c6fc1d6e82a4faf6dc6e0824299786ccc0/docs/segment-management-on-carbondata.md#query-data-with-specified-segments).
+IndexServer driver would have 2 important tasks, distributing the segments equally among the
+available executors and keeping track of the executor where the segment is cached.
+
+To achieve this 2 separate mappings would be maintained as follows.
+1. segment to executor location:
+This mapping will be maintained for each table and will enable the index server to track the
+cache location for each segment.
+
+2. Cache size held by each executor:
+ This mapping will be used to distribute the segments equally(on the basis of size) among the
+ executors.
+
+Once a request is received each segment would be iterated over and
+checked against tableToExecutorMapping to find if a executor is already
+assigned. If a mapping already exists then it means that most
+probably(if not evicted by LRU) the segment is already cached in that
+executor and the task for that segment has to be fired on this executor.
+
+If mapping is not found then first check executorToCacheMapping against
+the available executor list to find if any unassigned executor is
+present and use that executor for the current segment. If all the
+executors are assigned with some segment then find the least loaded
+executor on the basis of size.
+
+Initially the segment index size would be used to distribute the
+segments fairly among the executor because the actual cache size would
+be known to the driver only when the segments are cached and appropriate
+information is returned to the driver.
+
+**NOTE:** In case of legacy segment(version: 1.1) the index size is not available
+therefore all the legacy segments would be processed in a round robin
+fashion.
+
+After the job is completed the tasks would return the cache size held by
+each executor which would be updated to the executorToCacheMapping and
+the pruned blocklets which would be further used for result fetching.
+
+**Note:** Multiple JDBC drivers can connect to the index server to use the cache.
+
+## Reallocation of executor
+In case executor(s) become dead/unavailable then the segments that were
+earlier being handled by those would be reassigned to some other
+executor using the distribution logic.
+
+**Note:** Cache loading would be done again in the new executor for the
+current query.
+
+## MetaCache DDL
+The show metacache DDL has a new column called cache location will indicate whether the cache is
+from executor or driver. To drop cache the user has to enable/disable the index server using the
+dynamic configuration to clear the cache of the desired location.
+
+Refer: [MetaCacheDDL](https://github.com/apache/carbondata/blob/master/docs/ddl-of-carbondata.md#cache)
+
+## Fallback
+In case of any failure the index server would fallback to embedded mode
+which means that the JDBCServer would take care of distributed pruning.
+A similar job would be fired by the JDBCServer which would take care of
+pruning using its own executors. If for any reason the embedded mode
+also fails to prune the datamaps then the job would be passed on to
+driver.
+
+**NOTE:** In case of embedded mode a job would be fired after pruning to clear the
+cache as data cached in JDBCServer executors would be of no use.
+
+## Writing splits to a file
+If the response is too huge then it is better to write the splits to a file so that the driver can
+read this file and create the splits. This can be controlled using the property 'carbon.index.server
+.inmemory.serialization.threshold.inKB'. By default, the minimum value for this property is 0,
+meaning that no matter how small the splits are they would be written to the file. Maximum is
+102400KB which will mean if the size of the splits for a executor cross this value then they would
+be written to file.
+
+The user can set the location for these file by using 'carbon.indexserver.temp.path'. By default
+table path would be used to write the files.
+
+## Security
+The security for the index server is controlled through 'spark.carbon.indexserver.keytab' and 'spark
+.carbon.indexserver.principal'. These allow the RPC framework to login using the principal. It is
+recommended that the principal should be a super user, and the user should be exclusive for index
+server so that it does not grant access to any other service. Internally the operations would be
+executed as a Privileged Action using the login user.
+
+The Index Server is a long running service therefore the 'spark.yarn.keytab' and 'spark.yarn
+.principal' should be configured.
+
+## Configurations
+
+##### carbon.properties(JDBCServer)
+
+| Name | Default Value | Description |
+|:----------:|:-------------:|:------: |
+| carbon.enable.index.server | false | Enable the use of index server for pruning for the whole application. |
+| carbon.index.server.ip | NA | Specify the IP/HOST on which the server is started. Better to specify the private IP. |
+| carbon.index.server.port | NA | The port on which the index server is started. |
+| carbon.disable.index.server.fallback | false | Whether to enable/disable fallback for index server. Should be used for testing purposes only. Refer: [Fallback](#Fallback)|
+|carbon.index.server.max.jobname.length|NA|The max length of the job to show in the index server service UI. For bigger queries this may impact performance as the whole string would be sent from JDBCServer to IndexServer.|
+
+
+##### carbon.properties(IndexServer)
+
+| Name | Default Value | Description |
+|:----------:|:-------------:|:------: |
+| carbon.index.server.ip | NA | Specify the IP/HOST on which the server would be started. Better to specify the private IP. |
+| carbon.index.server.port | NA | The port on which the index server has to be started. |
+|carbon.index.server.max.worker.threads| 500 | Number of RPC handlers to open for accepting the requests from JDBC driver. Max accepted value is Integer.Max. Refer: [Hive configuration](https://github.com/apache/hive/blob/master/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java#L3441) |
+|carbon.max.executor.lru.cache.size| NA | Maximum memory **(in MB)** upto which the executor process can cache the data (DataMaps and reverse dictionary values). Only integer values greater than 0 are accepted. **NOTE:** Mandatory for the user to set. |
+|carbon.index.server.max.jobname.length|NA|The max length of the job to show in the index server application UI. For bigger queries this may impact performance as the whole string would be sent from JDBCServer to IndexServer.|
+|carbon.max.executor.threads.for.block.pruning|4| max executor threads used for block pruning. |
+|carbon.index.server.inmemory.serialization.threshold.inKB|300|Max in memory serialization size after reaching threshold data will be written to file. Min value that the user can set is 0KB and max is 102400KB. |
+|carbon.indexserver.temp.path|tablePath| The folder to write the split files if in memory datamap size for network transfers crossed the 'carbon.index.server.inmemory.serialization.threshold.inKB' limit.|
+
+
+##### spark-defaults.conf(only for secure mode)
+
+| Name | Default Value | Description |
+|:----------:|:-------------:|:------: |
+| spark.carbon.indexserver.principal | NA | Used for authentication, whether a valid service is trying to connect to the server or not. Set in both IndexServer and JDBCServer. |
+| spark.carbon.indexserver.keytab | NA | Specify the path to the keytab file through which authentication would happen. Set in both IndexServer and JDBCServer. |
+| spark.dynamicAllocation.enabled | true | Set to false, so that spark does not kill the executor, If executors are killed, cache would be lost. Applicable only for Index Server. |
+| spark.yarn.principal | NA | Should be set to the same user used for JDBCServer. Required only for IndexServer. |
+|spark.yarn.keytab| NA | Should be set to the same as JDBCServer. |
+
+##### spark-defaults.conf(non-secure mode)
+| Name | Default Value | Description |
+|:----------:|:-------------:|:------: |
+| spark.dynamicAllocation.enabled | true | Set to false, so that spark does not kill the executor, If executors are killed, cache would be lost. Applicable only for Index Server. |
+
+
+**NOTE:** Its better to create a new user for indexserver principal,
+that will authenticate the user to access the index server and no other service.
+
+##### core-site.xml
+
+| Name | Default Value | Description |
+|:----------:|:-------------:|:------: |
+| ipc.client.rpc-timeout.ms | NA | Set the above property to some appropriate value based on your estimated query time. The best option is to set this to the same value as spark.network.timeout. |
+
+##### dynamic-properties(set command)
+
+| Name | Default Value | Description |
+|:----------:|:-------------:|:------: |
+| carbon.enable.index.server | false | Enable the use of index server for pruning for the current session. |
+| carbon.enable.index.server.dbName.tableName | false | Enable the use of index server for the specified table in the current session. |
+
+
+## Starting the Server
+```
+./bin/spark-submit --master [yarn/local] --[o ptional parameters] --class org.apache.carbondata.indexserver.IndexServer [path to carbondata-spark2-<version>.jar]
+```
+Or
+```
+./sbin/start-indexserver.sh --master yarn --num-executors 2 /<absolute path>/carbondata-spark2-1.6.0.0100.jar
+```
+
+## FAQ
+
+Q. **Index Server is throwing Large response size exception.**
+
+A. The exception would show the size of response it is trying to send over the
+network. Use ipc.maximum.response.length to a value bigger than the
+response size.
+
+Q. **Index server is throwing Kerberos principal not set exception**
+
+A. Set spark.carbon.indexserver.principal to the correct principal in both IndexServer and
+JDBCServer configurations.
+
+Q. **Unable to connect to index server**
+
+A. Check whether the carbon.properties configurations are set in JDBCServer as well as the index
+server.
+
+Q. **IndexServer is throwing FileNotFoundException for index files.**
+
+A. Check whether the Index server and JDBCServer are connected to the
+same namenode or not. And the store should be shared by both
+
+Q. **OutOfMemoryException in DirectMemoryBuffer**
+
+A. Increase -XX:MaxDirectMemorySize in driver.extraJavaOptions to
+accommodate the large response in driver.
\ No newline at end of file
diff --git a/src/site/markdown/mv-datamap-guide.md b/src/site/markdown/mv-datamap-guide.md
new file mode 100644
index 0000000..d22357c
--- /dev/null
+++ b/src/site/markdown/mv-datamap-guide.md
@@ -0,0 +1,208 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to you under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# CarbonData MV DataMap
+
+* [Quick Example](#quick-example)
+* [MV DataMap](#mv-datamap-introduction)
+* [Loading Data](#loading-data)
+* [Querying Data](#querying-data)
+* [Compaction](#compacting-mv-tables)
+* [Data Management](#data-management-with-mv-tables)
+
+## Quick example
+
+Start spark-sql in terminal and run the following queries,
+```
+CREATE TABLE maintable(a int, b string, c int) stored by 'carbondata';
+insert into maintable select 1, 'ab', 2;
+CREATE DATAMAP datamap_1 on table maintable as SELECT a, sum(b) from maintable group by a;
+SELECT a, sum(b) from maintable group by a;
+// NOTE: run explain query and check if query hits the datamap table from the plan
+EXPLAIN SELECT a, sum(b) from maintable group by a;
+```
+
+## MV DataMap Introduction
+ MV tables are created as DataMaps and managed as tables internally by CarbonData. User can create
+ limitless MV datamaps on a table to improve query performance provided the storage requirements
+ and loading time is acceptable.
+
+ MV datamap can be a lazy or a non-lazy datamap. Once MV datamaps are created, CarbonData's
+ CarbonAnalyzer helps to select the most efficient MV datamap based on the user query and rewrite
+ the SQL to select the data from MV datamap instead of main table. Since the data size of MV
+ datamap is smaller and data is pre-processed, user queries are much faster.
+
+ For instance, main table called **sales** which is defined as
+
+ ```
+ CREATE TABLE sales (
+ order_time timestamp,
+ user_id string,
+ sex string,
+ country string,
+ quantity int,
+ price bigint)
+ STORED AS carbondata
+ ```
+
+ User can create MV tables using the Create DataMap DDL
+
+ ```
+ CREATE DATAMAP agg_sales
+ ON TABLE sales
+ USING "MV"
+ AS
+ SELECT country, sex, sum(quantity), avg(price)
+ FROM sales
+ GROUP BY country, sex
+ ```
+ **NOTE**:
+ * Group by/Filter columns has to be provided in projection list while creating mv datamap
+ * If only single parent table is involved in mv datamap creation, then TableProperties of Parent table
+ (if not present in a aggregate function like sum(col)) listed below will be
+ inherited to datamap table
+ 1. SORT_COLUMNS
+ 2. SORT_SCOPE
+ 3. TABLE_BLOCKSIZE
+ 4. FLAT_FOLDER
+ 5. LONG_STRING_COLUMNS
+ 6. LOCAL_DICTIONARY_ENABLE
+ 7. LOCAL_DICTIONARY_THRESHOLD
+ 8. LOCAL_DICTIONARY_EXCLUDE
+ 9. DICTIONARY_INCLUDE
+ 10. DICTIONARY_EXCLUDE
+ 11. INVERTED_INDEX
+ 12. NO_INVERTED_INDEX
+ 13. COLUMN_COMPRESSOR
+
+ * All columns of main table at once cannot participate in mv datamap table creation
+ * TableProperties can be provided in DMProperties excluding LOCAL_DICTIONARY_INCLUDE,
+ LOCAL_DICTIONARY_EXCLUDE, DICTIONARY_INCLUDE, DICTIONARY_EXCLUDE, INVERTED_INDEX,
+ NO_INVERTED_INDEX, SORT_COLUMNS, LONG_STRING_COLUMNS, RANGE_COLUMN & COLUMN_META_CACHE
+ * TableProperty given in DMProperties will be considered for mv creation, eventhough if same
+ property is inherited from parent table, which allows user to provide different tableproperties
+ for child table
+ * MV creation with limit or union all ctas queries is unsupported
+
+#### How MV tables are selected
+
+When a user query is submitted, during query planning phase, CarbonData will collect modular plan
+candidates and process the the ModularPlan based on registered summary data sets. Then,
+mv datamap table for this query will be selected among the candidates.
+
+For the main table **sales** and mv table **agg_sales** created above, following queries
+```
+SELECT country, sex, sum(quantity), avg(price) from sales GROUP BY country, sex
+
+SELECT sex, sum(quantity) from sales GROUP BY sex
+
+SELECT avg(price), country from sales GROUP BY country
+```
+
+will be transformed by CarbonData's query planner to query against mv table
+**agg_sales** instead of the main table **sales**
+
+However, for following queries
+```
+SELECT user_id, country, sex, sum(quantity), avg(price) from sales GROUP BY user_id, country, sex
+
+SELECT sex, avg(quantity) from sales GROUP BY sex
+
+SELECT country, max(price) from sales GROUP BY country
+```
+
+will query against main table **sales** only, because it does not satisfy mv table
+selection logic.
+
+## Loading data
+
+### Loading data to Non-Lazy MV Datamap
+
+In case of WITHOUT DEFERRED REBUILD, for existing table with loaded data, data load to MV table will
+be triggered by the CREATE DATAMAP statement when user creates the MV table.
+For incremental loads to main table, data to datamap will be loaded once the corresponding main
+table load is completed.
+
+### Loading data to Lazy MV Datamap
+
+In case of WITH DEFERRED REBUILD, data load to MV table will be triggered by the [Manual Refresh](./datamap-management.md#manual-refresh)
+command. MV datamap will be in DISABLED state in below scenarios,
+ * when mv datamap is created
+ * when data of main table and datamap are not in sync
+
+User should fire REBUILD DATAMAP command to sync all segments of main table with datamap table and
+which ENABLES the datamap for query
+
+### Loading data to Multiple MV's
+During load to main table, if anyone of the load to datamap table fails, then that corresponding
+datamap will be DISABLED and load to other datamaps mapped to main table will continue. User can
+fire REBUILD DATAMAP command to sync or else the subsequent table load will load the old failed
+loads along with current load and enable the disabled datamap.
+
+ **NOTE**:
+ * In case of InsertOverwrite/Update operation on parent table, all segments of datamap table will
+ be MARKED_FOR_DELETE and reload to datamap table will happen by REBUILD DATAMAP, in case of Lazy
+ mv datamap/ once InsertOverwrite/Update operation on parent table is finished, in case of
+ Non-Lazy mv.
+ * In case of full scan query, Data Size and Index Size of main table and child table will not the
+ same, as main table and child table has different column names.
+
+## Querying data
+As a technique for query acceleration, MV tables cannot be queried directly.
+Queries are to be made on main table. While doing query planning, internally CarbonData will check
+associated mv datamap tables with the main table, and do query plan transformation accordingly.
+
+User can verify whether a query can leverage mv datamap table or not by executing `EXPLAIN`
+command, which will show the transformed logical plan, and thus user can check whether mv datamap
+table is selected.
+
+
+## Compacting MV datamap
+
+### Compacting MV datamap table through Main Table compaction
+Running Compaction command (`ALTER TABLE COMPACT`)[COMPACTION TYPE-> MINOR/MAJOR] on main table will
+automatically compact the mv datamap tables created on the main table, once compaction on main table
+is done.
+
+### Compacting MV datamap table through DDL command
+Compaction on mv datamap can be triggered by running the following DDL command(supported only for mv).
+ ```
+ ALTER DATAMAP datamap_name COMPACT 'COMPACTION_TYPE'
+ ```
+
+## Data Management with mv tables
+In current implementation, data consistency needs to be maintained for both main table and mv datamap
+tables. Once there is mv datamap table created on the main table, following command on the main
+table is not supported:
+1. Data management command: `DELETE SEGMENT`.
+2. Schema management command: `ALTER TABLE DROP COLUMN`, `ALTER TABLE CHANGE DATATYPE`,
+ `ALTER TABLE RENAME`, `ALTER COLUMN RENAME`. Note that adding a new column is supported, and for
+ dropping columns and change datatype command, CarbonData will check whether it will impact the
+ mv datamap table, if not, the operation is allowed, otherwise operation will be rejected by
+ throwing exception.
+3. Partition management command: `ALTER TABLE ADD/DROP PARTITION`. Note that dropping a partition
+ will be allowed only if partition is participating in all datamaps associated with main table.
+ Drop Partition is not allowed, if any mv datamap is associated with more than one parent table.
+ Drop Partition directly on datamap table is not allowed.
+4. Complex Datatype's for mv datamap is not supported.
+
+However, there is still way to support these operations on main table, in current CarbonData
+release, user can do as following:
+1. Remove the mv datamap table by `DROP DATAMAP` command
+2. Carry out the data management operation on main table
+3. Create the mv datamap table again by `CREATE DATAMAP` command
+Basically, user can manually trigger the operation by re-building the datamap.
diff --git a/src/site/markdown/preaggregate-datamap-guide.md b/src/site/markdown/preaggregate-datamap-guide.md
index eff601d..5369bb7 100644
--- a/src/site/markdown/preaggregate-datamap-guide.md
+++ b/src/site/markdown/preaggregate-datamap-guide.md
@@ -176,6 +176,9 @@ It will show all DataMaps created on main table.
FROM sales
GROUP BY country, sex
```
+ **NOTE**:
+ * Preaggregate datamap is deprecated and it is replaced by MV datamap.
+ Please refer [CarbonData MV DataMap](./mv-datamap-guide.md) for more info.
#### Functions supported in pre-aggregate table
diff --git a/src/site/markdown/sdk-guide.md b/src/site/markdown/sdk-guide.md
index 573b595..b8e9f51 100644
--- a/src/site/markdown/sdk-guide.md
+++ b/src/site/markdown/sdk-guide.md
@@ -195,6 +195,7 @@ Each of SQL data types and Avro Data Types are mapped into data types of SDK. Fo
| BIGINT | LONG | DataTypes.LONG |
| DOUBLE | DOUBLE | DataTypes.DOUBLE |
| VARCHAR | - | DataTypes.STRING |
+| BINARY | - | DataTypes.BINARY |
| FLOAT | FLOAT | DataTypes.FLOAT |
| BYTE | - | DataTypes.BYTE |
| DATE | DATE | DataTypes.DATE |
@@ -223,7 +224,7 @@ Each of SQL data types and Avro Data Types are mapped into data types of SDK. Fo
The timestamp-micros logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one microsecond.
A timestamp-micros logical type annotates an Avro long, where the long stores the number of microseconds from the unix epoch, 1 January 1970 00:00:00.000000 UTC.
d. Decimal
- The decimal logical type represents an arbitrary-precision signed decimal number of the form unscaled × 10-scale.
+ The decimal logical type represents an arbitrary-precision signed decimal number of the form <em>unscaled × 10<sup>-scale</sup></em>.
A decimal logical type annotates Avro bytes or fixed types. The byte array must contain the two's-complement representation of the unscaled integer value in big-endian byte order. The scale is fixed, and is specified using an attribute.
e. Time (millisecond precision)
The time-millis logical type represents a time of day, with no reference to a particular calendar, time zone or date, with a precision of one millisecond.
@@ -382,6 +383,7 @@ public CarbonWriterBuilder withLoadOptions(Map<String, String> options);
* default value is null.
* l. inverted_index -- comma separated string columns for which inverted index needs to be
* generated
+ * m. table_page_size_inmb -- [1-1755] MB.
*
* @return updated CarbonWriterBuilder
*/
@@ -612,6 +614,10 @@ reader.close();
Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java) in the CarbonData repo.
+SDK reader also supports reading carbondata files and filling it to apache arrow vectors.
+Find example code at [ArrowCarbonReaderTest](https://github.com/apache/carbondata/blob/master/store/sdk/src/test/java/org/apache/carbondata/sdk/file/ArrowCarbonReaderTest.java) in the CarbonData repo.
+
+
## API List
### Class org.apache.carbondata.sdk.file.CarbonReader
@@ -685,6 +691,82 @@ public Object[] readNextBatchRow();
public void close();
```
+### Class org.apache.carbondata.sdk.file.ArrowCarbonReader
+```
+/**
+ * Carbon reader will fill the arrow vector after reading the carbondata files.
+ * This arrow byte[] can be used to create arrow table and used for in memory analytics
+ * Note: create a reader at blocklet level, so that arrow byte[] will not exceed INT_MAX
+ *
+ * @param carbonSchema org.apache.carbondata.sdk.file.Schema
+ * @return Serialized byte array
+ * @throws Exception
+ */
+public byte[] readArrowBatch(Schema carbonSchema) throws Exception;
+```
+
+```
+/**
+ * Carbon reader will fill the arrow vector after reading the carbondata files.
+ * This arrow byte[] can be used to create arrow table and used for in memory analytics
+ * Note: create a reader at blocklet level, so that arrow byte[] will not exceed INT_MAX
+ * User need to close the VectorSchemaRoot after usage by calling VectorSchemaRoot.close()
+ *
+ * @param carbonSchema org.apache.carbondata.sdk.file.Schema
+ * @return Arrow VectorSchemaRoot
+ * @throws Exception
+ */
+public VectorSchemaRoot readArrowVectors(Schema carbonSchema) throws Exception;
+```
+
+```
+/**
+ * Carbon reader will fill the arrow vector after reading carbondata files.
+ * Here unsafe memory address will be returned instead of byte[],
+ * so that this address can be sent across java to python or c modules and
+ * can directly read the content from this unsafe memory
+ * Note:Create a carbon reader at blocklet level using CarbonReader.buildWithSplits(split) method,
+ * so that arrow byte[] will not exceed INT_MAX.
+ *
+ * @param carbonSchema org.apache.carbondata.sdk.file.Schema
+ * @return address of the unsafe memory where arrow buffer is stored
+ * @throws Exception
+ */
+public long readArrowBatchAddress(Schema carbonSchema) throws Exception;
+```
+
+```
+/**
+ * Free the unsafe memory allocated , if unsafe arrow batch is used.
+ *
+ * @param address address of the unsafe memory where arrow bufferer is stored
+ */
+public void freeArrowBatchMemory(long address)
+```
+
+### Class org.apache.carbondata.sdk.file.arrow.ArrowConverter
+```
+/**
+ * To get the arrow vectors directly after filling from carbondata
+ *
+ * @return Arrow VectorSchemaRoot. which contains array of arrow vectors.
+ */
+public VectorSchemaRoot getArrowVectors() throws IOException;
+```
+
+```
+/**
+ * Utility API to convert back the arrow byte[] to arrow ArrowRecordBatch.
+ * User need to close the ArrowRecordBatch after usage by calling ArrowRecordBatch.close()
+ *
+ * @param batchBytes input byte array
+ * @param bufferAllocator arrow buffer allocator
+ * @return ArrowRecordBatch
+ * @throws IOException
+ */
+public static ArrowRecordBatch byteArrayToArrowBatch(byte[] batchBytes, BufferAllocator bufferAllocator) throws IOException;
+```
+
### Class org.apache.carbondata.sdk.file.CarbonReaderBuilder
```
/**
diff --git a/src/site/markdown/supported-data-types-in-carbondata.md b/src/site/markdown/supported-data-types-in-carbondata.md
index daf1acf..4960453 100644
--- a/src/site/markdown/supported-data-types-in-carbondata.md
+++ b/src/site/markdown/supported-data-types-in-carbondata.md
@@ -51,4 +51,4 @@
* Other Types
* BOOLEAN
-
+ * BINARY