You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2018/06/02 03:27:31 UTC
[4/5] carbondata-site git commit: doucment for latest release of
carbondata 1.4.0
doucment for latest release of carbondata 1.4.0
Project: http://git-wip-us.apache.org/repos/asf/carbondata-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata-site/commit/2941aaca
Tree: http://git-wip-us.apache.org/repos/asf/carbondata-site/tree/2941aaca
Diff: http://git-wip-us.apache.org/repos/asf/carbondata-site/diff/2941aaca
Branch: refs/heads/asf-site
Commit: 2941aacaca5b8278416dfbea8e1be4b27db33f5a
Parents: 8cb979c
Author: vandana7 <va...@gmail.com>
Authored: Wed May 30 18:38:20 2018 +0530
Committer: vandana7 <va...@gmail.com>
Committed: Wed May 30 18:38:20 2018 +0530
----------------------------------------------------------------------
content/WEB-INF/classes/application.conf | 4 +-
content/bloomfilter-datamap-guide.html | 306 +++++++
content/configuration-parameters.html | 8 +
content/data-management-on-carbondata.html | 3 +
content/datamap-developer-guide.html | 3 +
content/faq.html | 3 +
content/file-structure-of-carbondata.html | 3 +
content/index.html | 3 +
content/installation-guide.html | 3 +
content/lucene-datamap-guide.html | 337 ++++++++
content/mainpage.html | 16 +-
content/pdf/CarbonData Documentation.pdf | Bin 264034 -> 292327 bytes
content/preaggregate-datamap-guide.html | 3 +
content/quick-start-guide.html | 3 +
content/sdk-guide.html | 820 +++++++++++++++++++
content/sdk-writer-guide.html | 3 +
content/security.html | 3 +
content/streaming-guide.html | 15 +-
content/supported-data-types-in-carbondata.html | 4 +
content/timeseries-datamap-guide.html | 3 +
content/troubleshooting.html | 3 +
content/useful-tips-on-carbondata.html | 3 +
content/videogallery.html | 3 +
src/main/resources/application.conf | 4 +-
src/main/scala/html/header.html | 3 +
src/main/webapp/bloomfilter-datamap-guide.html | 306 +++++++
src/main/webapp/configuration-parameters.html | 8 +
.../webapp/data-management-on-carbondata.html | 3 +
src/main/webapp/datamap-developer-guide.html | 3 +
src/main/webapp/faq.html | 3 +
.../webapp/file-structure-of-carbondata.html | 3 +
src/main/webapp/index.html | 10 +
src/main/webapp/installation-guide.html | 3 +
src/main/webapp/lucene-datamap-guide.html | 337 ++++++++
src/main/webapp/mainpage.html | 16 +-
.../webapp/pdf/CarbonData Documentation.pdf | Bin 264034 -> 292327 bytes
src/main/webapp/preaggregate-datamap-guide.html | 3 +
src/main/webapp/quick-start-guide.html | 3 +
src/main/webapp/sdk-guide.html | 820 +++++++++++++++++++
src/main/webapp/sdk-writer-guide.html | 3 +
src/main/webapp/security.html | 3 +
src/main/webapp/streaming-guide.html | 15 +-
.../supported-data-types-in-carbondata.html | 4 +
src/main/webapp/timeseries-datamap-guide.html | 3 +
src/main/webapp/troubleshooting.html | 3 +
src/main/webapp/useful-tips-on-carbondata.html | 3 +
src/main/webapp/videogallery.html | 3 +
src/site/markdown/bloomfilter-datamap-guide.md | 98 +++
src/site/markdown/configuration-parameters.md | 3 +-
.../markdown/data-management-on-carbondata.md | 4 +-
src/site/markdown/lucene-datamap-guide.md | 159 ++++
src/site/markdown/sdk-guide.md | 680 +++++++++++++++
src/site/markdown/sdk-writer-guide.md | 359 --------
src/site/markdown/streaming-guide.md | 12 +-
.../supported-data-types-in-carbondata.md | 2 +
src/site/markdown/timeseries-datamap-guide.md | 8 +-
src/site/pdf.xml | 8 +-
57 files changed, 4044 insertions(+), 403 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/WEB-INF/classes/application.conf
----------------------------------------------------------------------
diff --git a/content/WEB-INF/classes/application.conf b/content/WEB-INF/classes/application.conf
index ef8c3dc..df5280b 100644
--- a/content/WEB-INF/classes/application.conf
+++ b/content/WEB-INF/classes/application.conf
@@ -8,10 +8,12 @@ fileList=["configuration-parameters",
"supported-data-types-in-carbondata",
"troubleshooting",
"useful-tips-on-carbondata",
- "sdk-writer-guide",
+ "sdk-guide",
"datamap-developer-guide"
]
dataMapFileList=[
+ "bloomfilter-datamap-guide",
+ "lucene-datamap-guide",
"preaggregate-datamap-guide",
"timeseries-datamap-guide"]
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/bloomfilter-datamap-guide.html
----------------------------------------------------------------------
diff --git a/content/bloomfilter-datamap-guide.html b/content/bloomfilter-datamap-guide.html
new file mode 100644
index 0000000..41aef5c
--- /dev/null
+++ b/content/bloomfilter-datamap-guide.html
@@ -0,0 +1,306 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'>
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <title>CarbonData</title>
+ <style>
+
+ </style>
+ <!-- Bootstrap -->
+
+ <link rel="stylesheet" href="css/bootstrap.min.css">
+ <link href="css/style.css" rel="stylesheet">
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="js/jquery.min.js"></script>
+ <script src="js/bootstrap.min.js"></script>
+
+
+</head>
+<body>
+<header>
+ <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper">
+ <div class="container">
+ <div class="navbar-header">
+ <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse"
+ class="navbar-toggle collapsed" type="button">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="index.html" class="logo">
+ <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/>
+ </a>
+ </div>
+ <div class="navbar-collapse collapse cd_navcontnt" id="navbar">
+ <ul class="nav navbar-nav navbar-right navlist-custom">
+ <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a>
+ </li>
+ <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false"> Download <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
+ target="_blank">Apache CarbonData 1.3.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
+ target="_blank">Apache CarbonData 1.3.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
+ target="_blank">Apache CarbonData 1.2.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
+ target="_blank">Apache CarbonData 1.1.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
+ target="_blank">Apache CarbonData 1.1.0</a></li>
+ <li>
+ <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
+ target="_blank">Apache CarbonData 1.0.0</a></li>
+ <li>
+ <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
+ target="_blank">Apache CarbonData 0.2.0</a></li>
+ <li>
+ <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
+ target="_blank">Apache CarbonData 0.1.1</a></li>
+ <li>
+ <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
+ target="_blank">Apache CarbonData 0.1.0</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
+ target="_blank">Release Archive</a></li>
+ </ul>
+ </li>
+ <li><a href="mainpage.html" class="active">Documentation</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false">Community <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"
+ target="_blank">Contributing to CarbonData</a></li>
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"
+ target="_blank">Release Guide</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"
+ target="_blank">Project PMC and Committers</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"
+ target="_blank">CarbonData Meetups</a></li>
+ <li><a href="security.html">Apache CarbonData Security</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/CARBONDATA" target="_blank">Apache
+ Jira</a></li>
+ <li><a href="videogallery.html">CarbonData Videos </a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="apache_link hidden-xs dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="hidden-lg hidden-md hidden-sm dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li>
+ <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a>
+
+ </li>
+
+ </ul>
+ </div><!--/.nav-collapse -->
+ <div id="search-box">
+ <form method="get" action="http://www.google.com/search" target="_blank">
+ <div class="search-block">
+ <table border="0" cellpadding="0" width="100%">
+ <tr>
+ <td style="width:80%">
+ <input type="text" name="q" size=" 5" maxlength="255" value=""
+ class="search-input" placeholder="Search...." required/>
+ </td>
+ <td style="width:20%">
+ <input type="submit" value="Search"/></td>
+ </tr>
+ <tr>
+ <td align="left" style="font-size:75%" colspan="2">
+ <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/>
+ <span style=" position: relative; top: -3px;"> Only search for CarbonData</span>
+ </td>
+ </tr>
+ </table>
+ </div>
+ </form>
+ </div>
+ </div>
+ </nav>
+</header> <!-- end Header part -->
+
+<div class="fixed-padding"></div> <!-- top padding with fixde header -->
+
+<section><!-- Dashboard nav -->
+ <div class="container-fluid q">
+ <div class="col-sm-12 col-md-12 maindashboard">
+ <div class="row">
+ <section>
+ <div style="padding:10px 15px;">
+ <div id="viewpage" name="viewpage">
+ <div class="row">
+ <div class="col-sm-12 col-md-12">
+ <div><h1>
+<a id="carbondata-bloomfilter-datamap-alpha-feature-in-140" class="anchor" href="#carbondata-bloomfilter-datamap-alpha-feature-in-140" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData BloomFilter DataMap (Alpha feature in 1.4.0)</h1>
+<ul>
+<li><a href="#datamap-management">DataMap Management</a></li>
+<li><a href="#bloomfilter-datamap-introduction">BloomFilter Datamap Introduction</a></li>
+<li><a href="#loading-data">Loading Data</a></li>
+<li><a href="#querying-data">Querying Data</a></li>
+<li><a href="#data-management-with-bloomfilter-datamap">Data Management</a></li>
+</ul>
+<h4>
+<a id="datamap-management" class="anchor" href="#datamap-management" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>DataMap Management</h4>
+<p>Creating BloomFilter DataMap</p>
+<pre><code>CREATE DATAMAP [IF NOT EXISTS] datamap_name
+ON TABLE main_table
+USING 'bloomfilter'
+DMPROPERTIES ('index_columns'='city, name', 'BLOOM_SIZE'='640000', 'BLOOM_FPP'='0.00001')
+</code></pre>
+<p>Dropping specified datamap</p>
+<pre><code>DROP DATAMAP [IF EXISTS] datamap_name
+ON TABLE main_table
+</code></pre>
+<p>Showing all DataMaps on this table</p>
+<pre><code>SHOW DATAMAP
+ON TABLE main_table
+</code></pre>
+<p>It will show all DataMaps created on main table.</p>
+<h2>
+<a id="bloomfilter-datamap-introduction" class="anchor" href="#bloomfilter-datamap-introduction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>BloomFilter DataMap Introduction</h2>
+<p>A Bloom filter is a space-efficient probabilistic data structure that is used to test whether an element is a member of a set.
+Carbondata introduce BloomFilter as an index datamap to enhance the performance of querying with precise value.
+It is well suitable for queries that do precise match on high cardinality columns(such as Name/ID).
+Internally, CarbonData maintains a BloomFilter per blocklet for each index column to indicate that whether a value of the column is in this blocklet.
+Just like the other datamaps, BloomFilter datamap is managed ablong with main tables by CarbonData.
+User can create BloomFilter datamap on specified columns with specified BloomFilter configurations such as size and probability.</p>
+<p>For instance, main table called <strong>datamap_test</strong> which is defined as:</p>
+<pre><code>CREATE TABLE datamap_test (
+ id string,
+ name string,
+ age int,
+ city string,
+ country string)
+STORED BY 'carbondata'
+TBLPROPERTIES('SORT_COLUMNS'='id')
+</code></pre>
+<p>In the above example, <code>id</code> and <code>name</code> are high cardinality columns
+and we always query on <code>id</code> and <code>name</code> with precise value.
+since <code>id</code> is in the sort_columns and it is orderd,
+query on it will be fast because CarbonData can skip all the irrelative blocklets.
+But queries on <code>name</code> may be bad since the blocklet minmax may not help,
+because in each blocklet the range of the value of <code>name</code> may be the same -- all from A<em>~z</em>.
+In this case, user can create a BloomFilter datamap on column <code>name</code>.
+Moreover, user can also create a BloomFilter datamap on the sort_columns.
+This is useful if user has too many segments and the range of the value of sort_columns are almost the same.</p>
+<p>User can create BloomFilter datamap using the Create DataMap DDL:</p>
+<pre><code>CREATE DATAMAP dm
+ON TABLE datamap_test
+USING 'bloomfilter'
+DMPROPERTIES ('INDEX_COLUMNS' = 'name,id', 'BLOOM_SIZE'='640000', 'BLOOM_FPP'='0.00001', 'BLOOM_COMPRESS'='true')
+</code></pre>
+<p><strong>Properties for BloomFilter DataMap</strong></p>
+<table>
+<thead>
+<tr>
+<th>Property</th>
+<th>Is Required</th>
+<th>Default Value</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>INDEX_COLUMNS</td>
+<td>YES</td>
+<td></td>
+<td>Carbondata will generate BloomFilter index on these columns. Queries on there columns are usually like 'COL = VAL'.</td>
+</tr>
+<tr>
+<td>BLOOM_SIZE</td>
+<td>NO</td>
+<td>32000</td>
+<td>This value is internally used by BloomFilter as the number of expected insertions, it will affects the size of BloomFilter index. Since each blocklet has a BloomFilter here, so the value is the approximate records in a blocklet. In another word, the value 32000 * #noOfPagesInBlocklet. The value should be an integer.</td>
+</tr>
+<tr>
+<td>BLOOM_FPP</td>
+<td>NO</td>
+<td>0.01</td>
+<td>This value is internally used by BloomFilter as the False-Positive Probability, it will affects the size of bloomfilter index as well as the number of hash functions for the BloomFilter. The value should be in range (0, 1).</td>
+</tr>
+<tr>
+<td>BLOOM_COMPRESS</td>
+<td>NO</td>
+<td>true</td>
+<td>Whether to compress the BloomFilter index files.</td>
+</tr>
+</tbody>
+</table>
+<h2>
+<a id="loading-data" class="anchor" href="#loading-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading Data</h2>
+<p>When loading data to main table, BloomFilter files will be generated for all the
+index_columns given in DMProperties which contains the blockletId and a BloomFilter for each index column.
+These index files will be written inside a folder named with datamap name
+inside each segment folders.</p>
+<h2>
+<a id="querying-data" class="anchor" href="#querying-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Querying Data</h2>
+<p>A system level configuration <code>carbon.query.datamap.bloom.cache.size</code> can used to enhance query performance with BloomFilter datamap by providing a cache for the bloomfilter index files.
+The default value is <code>512</code> and its unit is <code>MB</code>. Internally the cache will be expired after it's idle for 2 hours.</p>
+<p>User can verify whether a query can leverage BloomFilter datamap by executing <code>EXPLAIN</code> command,
+which will show the transformed logical plan, and thus user can check whether the BloomFilter datamap can skip blocklets during the scan.
+If the datamap does not prune blocklets well, you can try to increase the value of property <code>BLOOM_SIZE</code> and decrease the value of property <code>BLOOM_FPP</code>.</p>
+<h2>
+<a id="data-management-with-bloomfilter-datamap" class="anchor" href="#data-management-with-bloomfilter-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Data Management With BloomFilter DataMap</h2>
+<p>Data management with BloomFilter datamap has no difference with that on Lucene datamap. You can refer to the corresponding section in <code>CarbonData BloomFilter DataMap</code>.</p>
+</div>
+</div>
+</div>
+</div>
+<div class="doc-footer">
+ <a href="#top" class="scroll-top">Top</a>
+</div>
+</div>
+</section>
+</div>
+</div>
+</div>
+</section><!-- End systemblock part -->
+<script src="js/custom.js"></script>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/configuration-parameters.html
----------------------------------------------------------------------
diff --git a/content/configuration-parameters.html b/content/configuration-parameters.html
index 8db28db..2768b0e 100644
--- a/content/configuration-parameters.html
+++ b/content/configuration-parameters.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>
@@ -590,6 +593,11 @@
<td>0.8</td>
<td>Specifies the minimum resource (executor) ratio needed for starting the block distribution. The default value is 0.8, which indicates 80% of the requested resource is allocated for starting block distribution. The minimum value is 0.1 min and the maximum value is 1.0.</td>
</tr>
+<tr>
+<td>carbon.search.enabled</td>
+<td>false</td>
+<td>If set to true, it will use CarbonReader to do distributed scan directly instead of using compute framework like spark, thus avoiding limitation of compute framework like SQL optimizer and task scheduling overhead.</td>
+</tr>
</tbody>
</table>
<ul>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/data-management-on-carbondata.html
----------------------------------------------------------------------
diff --git a/content/data-management-on-carbondata.html b/content/data-management-on-carbondata.html
index 49f2fe8..a9c7926 100644
--- a/content/data-management-on-carbondata.html
+++ b/content/data-management-on-carbondata.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/datamap-developer-guide.html
----------------------------------------------------------------------
diff --git a/content/datamap-developer-guide.html b/content/datamap-developer-guide.html
index 18ec74d..367fcb7 100644
--- a/content/datamap-developer-guide.html
+++ b/content/datamap-developer-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/faq.html
----------------------------------------------------------------------
diff --git a/content/faq.html b/content/faq.html
index 517fea2..5d43ff1 100644
--- a/content/faq.html
+++ b/content/faq.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/file-structure-of-carbondata.html
----------------------------------------------------------------------
diff --git a/content/file-structure-of-carbondata.html b/content/file-structure-of-carbondata.html
index 5ea9cbb..dc1e239 100644
--- a/content/file-structure-of-carbondata.html
+++ b/content/file-structure-of-carbondata.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/index.html
----------------------------------------------------------------------
diff --git a/content/index.html b/content/index.html
index 3316f63..c201fb2 100644
--- a/content/index.html
+++ b/content/index.html
@@ -54,6 +54,9 @@
class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/installation-guide.html
----------------------------------------------------------------------
diff --git a/content/installation-guide.html b/content/installation-guide.html
index 6d944f7..ffb750e 100644
--- a/content/installation-guide.html
+++ b/content/installation-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/lucene-datamap-guide.html
----------------------------------------------------------------------
diff --git a/content/lucene-datamap-guide.html b/content/lucene-datamap-guide.html
new file mode 100644
index 0000000..5f56429
--- /dev/null
+++ b/content/lucene-datamap-guide.html
@@ -0,0 +1,337 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'>
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <title>CarbonData</title>
+ <style>
+
+ </style>
+ <!-- Bootstrap -->
+
+ <link rel="stylesheet" href="css/bootstrap.min.css">
+ <link href="css/style.css" rel="stylesheet">
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="js/jquery.min.js"></script>
+ <script src="js/bootstrap.min.js"></script>
+
+
+</head>
+<body>
+<header>
+ <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper">
+ <div class="container">
+ <div class="navbar-header">
+ <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse"
+ class="navbar-toggle collapsed" type="button">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="index.html" class="logo">
+ <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/>
+ </a>
+ </div>
+ <div class="navbar-collapse collapse cd_navcontnt" id="navbar">
+ <ul class="nav navbar-nav navbar-right navlist-custom">
+ <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a>
+ </li>
+ <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false"> Download <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
+ target="_blank">Apache CarbonData 1.3.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
+ target="_blank">Apache CarbonData 1.3.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
+ target="_blank">Apache CarbonData 1.2.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
+ target="_blank">Apache CarbonData 1.1.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
+ target="_blank">Apache CarbonData 1.1.0</a></li>
+ <li>
+ <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
+ target="_blank">Apache CarbonData 1.0.0</a></li>
+ <li>
+ <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
+ target="_blank">Apache CarbonData 0.2.0</a></li>
+ <li>
+ <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
+ target="_blank">Apache CarbonData 0.1.1</a></li>
+ <li>
+ <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
+ target="_blank">Apache CarbonData 0.1.0</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
+ target="_blank">Release Archive</a></li>
+ </ul>
+ </li>
+ <li><a href="mainpage.html" class="active">Documentation</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false">Community <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"
+ target="_blank">Contributing to CarbonData</a></li>
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"
+ target="_blank">Release Guide</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"
+ target="_blank">Project PMC and Committers</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"
+ target="_blank">CarbonData Meetups</a></li>
+ <li><a href="security.html">Apache CarbonData Security</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/CARBONDATA" target="_blank">Apache
+ Jira</a></li>
+ <li><a href="videogallery.html">CarbonData Videos </a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="apache_link hidden-xs dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="hidden-lg hidden-md hidden-sm dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li>
+ <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a>
+
+ </li>
+
+ </ul>
+ </div><!--/.nav-collapse -->
+ <div id="search-box">
+ <form method="get" action="http://www.google.com/search" target="_blank">
+ <div class="search-block">
+ <table border="0" cellpadding="0" width="100%">
+ <tr>
+ <td style="width:80%">
+ <input type="text" name="q" size=" 5" maxlength="255" value=""
+ class="search-input" placeholder="Search...." required/>
+ </td>
+ <td style="width:20%">
+ <input type="submit" value="Search"/></td>
+ </tr>
+ <tr>
+ <td align="left" style="font-size:75%" colspan="2">
+ <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/>
+ <span style=" position: relative; top: -3px;"> Only search for CarbonData</span>
+ </td>
+ </tr>
+ </table>
+ </div>
+ </form>
+ </div>
+ </div>
+ </nav>
+</header> <!-- end Header part -->
+
+<div class="fixed-padding"></div> <!-- top padding with fixde header -->
+
+<section><!-- Dashboard nav -->
+ <div class="container-fluid q">
+ <div class="col-sm-12 col-md-12 maindashboard">
+ <div class="row">
+ <section>
+ <div style="padding:10px 15px;">
+ <div id="viewpage" name="viewpage">
+ <div class="row">
+ <div class="col-sm-12 col-md-12">
+ <div><h1>
+<a id="carbondata-lucene-datamap-alpha-feature-in-140" class="anchor" href="#carbondata-lucene-datamap-alpha-feature-in-140" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Lucene DataMap (Alpha feature in 1.4.0)</h1>
+<ul>
+<li><a href="#datamap-management">DataMap Management</a></li>
+<li><a href="#lucene-datamap-introduction">Lucene Datamap</a></li>
+<li><a href="#loading-data">Loading Data</a></li>
+<li><a href="#querying-data">Querying Data</a></li>
+<li><a href="#data-management-with-lucene-datamap">Data Management</a></li>
+</ul>
+<h4>
+<a id="datamap-management" class="anchor" href="#datamap-management" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>DataMap Management</h4>
+<p>Lucene DataMap can be created using following DDL</p>
+<pre><code>CREATE DATAMAP [IF NOT EXISTS] datamap_name
+ON TABLE main_table
+USING 'lucene'
+DMPROPERTIES ('index_columns'='city, name', ...)
+</code></pre>
+<p>DataMap can be dropped using following DDL:</p>
+<pre><code>DROP DATAMAP [IF EXISTS] datamap_name
+ON TABLE main_table
+</code></pre>
+<p>To show all DataMaps created, use:</p>
+<pre><code>SHOW DATAMAP
+ON TABLE main_table
+</code></pre>
+<p>It will show all DataMaps created on main table.</p>
+<h2>
+<a id="lucene-datamap-introduction" class="anchor" href="#lucene-datamap-introduction" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Lucene DataMap Introduction</h2>
+<p>Lucene is a high performance, full featured text search engine. Lucene is integrated to carbon as
+an index datamap and managed along with main tables by CarbonData.User can create lucene datamap
+to improve query performance on string columns which has content of more length. So, user can
+search tokenized word or pattern of it using lucene query on text content.</p>
+<p>For instance, main table called <strong>datamap_test</strong> which is defined as:</p>
+<pre><code>CREATE TABLE datamap_test (
+ name string,
+ age int,
+ city string,
+ country string)
+STORED BY 'carbondata'
+</code></pre>
+<p>User can create Lucene datamap using the Create DataMap DDL:</p>
+<pre><code>CREATE DATAMAP dm
+ON TABLE datamap_test
+USING 'lucene'
+DMPROPERTIES ('INDEX_COLUMNS' = 'name, country',)
+</code></pre>
+<p><strong>DMProperties</strong></p>
+<ol>
+<li>INDEX_COLUMNS: The list of string columns on which lucene creates indexes.</li>
+<li>FLUSH_CACHE: size of the cache to maintain in Lucene writer, if specified then it tries to
+aggregate the unique data till the cache limit and flush to Lucene. It is best suitable for low
+cardinality dimensions.</li>
+<li>SPLIT_BLOCKLET: when made as true then store the data in blocklet wise in lucene , it means new
+folder will be created for each blocklet, thus, it eliminates storing blockletid in lucene and
+also it makes lucene small chunks of data.</li>
+</ol>
+<h2>
+<a id="loading-data" class="anchor" href="#loading-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Loading data</h2>
+<p>When loading data to main table, lucene index files will be generated for all the
+index_columns(String Columns) given in DMProperties which contains information about the data
+location of index_columns. These index files will be written inside a folder named with datamap name
+inside each segment folders.</p>
+<p>A system level configuration carbon.lucene.compression.mode can be added for best compression of
+lucene index files. The default value is speed, where the index writing speed will be more. If the
+value is compression, the index file size will be compressed.</p>
+<h2>
+<a id="querying-data" class="anchor" href="#querying-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Querying data</h2>
+<p>As a technique for query acceleration, Lucene indexes cannot be queried directly.
+Queries are to be made on main table. when a query with TEXT_MATCH('name:c10') or
+TEXT_MATCH_WITH_LIMIT('name:n10',10)[the second parameter represents the number of result to be
+returned, if user does not specify this value, all results will be returned without any limit] is
+fired, two jobs are fired.The first job writes the temporary files in folder created at table level
+which contains lucene's seach results and these files will be read in second job to give faster
+results. These temporary files will be cleared once the query finishes.</p>
+<p>User can verify whether a query can leverage Lucene datamap or not by executing <code>EXPLAIN</code>
+command, which will show the transformed logical plan, and thus user can check whether TEXT_MATCH()
+filter is applied on query or not.</p>
+<p><strong>Note:</strong></p>
+<ol>
+<li>
+<p>The filter columns in TEXT_MATCH or TEXT_MATCH_WITH_LIMIT must be always in lower case and
+filter condition like 'AND','OR' must be in upper case.</p>
+<p>Ex:</p>
+<pre><code>select * from datamap_test where TEXT_MATCH('name:*10 AND name:*n*')
+</code></pre>
+</li>
+<li>
+<p>Query supports only one TEXT_MATCH udf for filter condition and not multiple udfs.</p>
+<p>The following query is supported:</p>
+<pre><code>select * from datamap_test where TEXT_MATCH('name:*10 AND name:*n*')
+</code></pre>
+<p>The following query is not supported:</p>
+<pre><code>select * from datamap_test where TEXT_MATCH('name:*10) AND TEXT_MATCH(name:*n*')
+</code></pre>
+</li>
+</ol>
+<p>Below like queries can be converted to text_match queries as following:</p>
+<pre><code>select * from datamap_test where name='n10'
+
+select * from datamap_test where name like 'n1%'
+
+select * from datamap_test where name like '%10'
+
+select * from datamap_test where name like '%n%'
+
+select * from datamap_test where name like '%10' and name not like '%n%'
+</code></pre>
+<p>Lucene TEXT_MATCH Queries:</p>
+<pre><code>select * from datamap_test where TEXT_MATCH('name:n10')
+
+select * from datamap_test where TEXT_MATCH('name:n1*')
+
+select * from datamap_test where TEXT_MATCH('name:*10')
+
+select * from datamap_test where TEXT_MATCH('name:*n*')
+
+select * from datamap_test where TEXT_MATCH('name:*10 -name:*n*')
+</code></pre>
+<p><strong>Note:</strong> For lucene queries and syntax, refer to <a href="www.lucenetutorial.com/lucene-query-syntax.html">lucene-syntax</a></p>
+<h2>
+<a id="data-management-with-lucene-datamap" class="anchor" href="#data-management-with-lucene-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Data Management with lucene datamap</h2>
+<p>Once there is lucene datamap is created on the main table, following command on the main
+table
+is not supported:</p>
+<ol>
+<li>Data management command: <code>UPDATE/DELETE</code>.</li>
+<li>Schema management command: <code>ALTER TABLE DROP COLUMN</code>, <code>ALTER TABLE CHANGE DATATYPE</code>,
+<code>ALTER TABLE RENAME</code>.</li>
+</ol>
+<p><strong>Note</strong>: Adding a new column is supported, and for dropping columns and change datatype
+command, CarbonData will check whether it will impact the lucene datamap, if not, the operation
+is allowed, otherwise operation will be rejected by throwing exception.</p>
+<ol start="3">
+<li>Partition management command: <code>ALTER TABLE ADD/DROP PARTITION</code>.</li>
+</ol>
+<p>However, there is still way to support these operations on main table, in current CarbonData
+release, user can do as following:</p>
+<ol>
+<li>Remove the lucene datamap by <code>DROP DATAMAP</code> command.</li>
+<li>Carry out the data management operation on main table.</li>
+<li>Create the lucene datamap again by <code>CREATE DATAMAP</code> command.
+Basically, user can manually trigger the operation by re-building the datamap.</li>
+</ol>
+</div>
+</div>
+</div>
+</div>
+<div class="doc-footer">
+ <a href="#top" class="scroll-top">Top</a>
+</div>
+</div>
+</section>
+</div>
+</div>
+</div>
+</section><!-- End systemblock part -->
+<script src="js/custom.js"></script>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/mainpage.html
----------------------------------------------------------------------
diff --git a/content/mainpage.html b/content/mainpage.html
index 7a5501b..d2746db 100644
--- a/content/mainpage.html
+++ b/content/mainpage.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>
@@ -188,17 +191,16 @@
<div>
<ul class="sub-nav">
<li><a href="quick-start-guide.html">Quick Start</a></li>
- <li><a href="file-structure-of-carbondata.html">CarbonData
- File Structure</a></li>
- <li><a href="supported-data-types-in-carbondata.html">Data
- Types</a></li>
- <li><a href="data-management-on-carbondata.html">Data
- Management On CarbonData</a></li>
+ <li><a href="file-structure-of-carbondata.html">CarbonData File Structure</a></li>
+ <li><a href="supported-data-types-in-carbondata.html">Data Types</a></li>
+ <li><a href="data-management-on-carbondata.html">Data Management On CarbonData</a></li>
<li><a href="installation-guide.html">Installation Guide</a></li>
<li><a href="configuration-parameters.html">Configuring CarbonData</a></li>
<li><a href="streaming-guide.html">Streaming Guide</a></li>
- <li><a href="sdk-writer-guide.html">SDK Writer Guide</a></li>
+ <li><a href="sdk-guide.html">SDK Writer Guide</a></li>
<li><a href="datamap-developer-guide.html">DataMap Developer Guide</a></li>
+ <li><a href="bloomfilter-datamap-guide.html">CarbonData Pre-aggregate DataMap</a></li>
+ <li><a href="lucene-datamap-guide.html">CarbonData Pre-aggregate DataMap</a></li>
<li><a href="preaggregate-datamap-guide.html">CarbonData Pre-aggregate DataMap</a></li>
<li><a href="timeseries-datamap-guide.html">CarbonData Timeseries DataMap</a></li>
<li><a href="faq.html">FAQs</a></li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/pdf/CarbonData Documentation.pdf
----------------------------------------------------------------------
diff --git a/content/pdf/CarbonData Documentation.pdf b/content/pdf/CarbonData Documentation.pdf
index 544de04..a557cca 100644
Binary files a/content/pdf/CarbonData Documentation.pdf and b/content/pdf/CarbonData Documentation.pdf differ
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/preaggregate-datamap-guide.html
----------------------------------------------------------------------
diff --git a/content/preaggregate-datamap-guide.html b/content/preaggregate-datamap-guide.html
index 35a0f62..d195bfe 100644
--- a/content/preaggregate-datamap-guide.html
+++ b/content/preaggregate-datamap-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/2941aaca/content/quick-start-guide.html
----------------------------------------------------------------------
diff --git a/content/quick-start-guide.html b/content/quick-start-guide.html
index 002df87..20cfbf1 100644
--- a/content/quick-start-guide.html
+++ b/content/quick-start-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
target="_blank">Apache CarbonData 1.3.1</a></li>
<li>