You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2018/08/20 09:17:32 UTC
[4/8] carbondata-site git commit: CarbonData 1.4.1 Release updated
including the release package, documentation,
and other required link. PDF generation is been removed
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/index.html
----------------------------------------------------------------------
diff --git a/content/index.html b/content/index.html
index 3fe3ba4..e4e0c0e 100644
--- a/content/index.html
+++ b/content/index.html
@@ -54,6 +54,9 @@
class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -63,27 +66,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
@@ -328,6 +310,13 @@
</h4>
<div class="linkblock">
<div class="block-row">
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a>
+ <span class="release-date">Aug 2018</span>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.4.1+Release"
+ class="whatsnew" target="_blank">what's new</a>
+ </div>
+ <div class="block-row">
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a>
<span class="release-date">May 2018</span>
@@ -348,21 +337,6 @@
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.3.0+Release"
class="whatsnew" target="_blank">what's new</a>
</div>
- <div class="block-row">
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a>
- <span class="release-date">Sept 2017</span>
- <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.2.0+Release"
- class="whatsnew" target="_blank">what's new</a>
- </div>
- <div class="block-row">
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a>
- <span class="release-date">July 2017</span>
- <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.1.1+Release"
- class="whatsnew" target="_blank">what's new</a>
- </div>
- </div>
</div>
<div class="nextR">
<h4 class="title">Release Notes
@@ -389,10 +363,6 @@
class="fa fa-github icon-margin-r" aria-hidden="true"></i>
CarbonData Github
</a>
- <a href="pdf/CarbonData Documentation.pdf" class="quickstart-btn"
- target="_blank"><i class="fa fa-file-pdf-o icon-margin-r"
- aria-hidden="true"></i></i> Download Documentation
- </a>
<a href="pdf/CarbonData-TPCH-Report.pdf" class="quickstart-btn"
target="_blank"><i class="fa fa-file-pdf-o icon-margin-r"
aria-hidden="true"></i></i> CarbonData TPCH Report
@@ -498,7 +468,7 @@
to do is:</p>
<ol class="orderlist">
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
target="_blank">Download</a>the latest release.
</li>
@@ -689,7 +659,7 @@
<div class="bottomcontant">
<div class="container-fluid">
<div class="col-md-8 col-sm-8">
- <p class="copyright-txt">Copyright © 2016-2017 Apache CarbonData. All rights reserved |
+ <p class="copyright-txt">Copyright © 2017-2018 Apache CarbonData. All rights reserved |
<a href="http://www.apache.org/" class="term-links" target="_blank">Apache Software
Foundation </a> |
<a href="http://www.apache.org/foundation/policies/privacy.html"
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/installation-guide.html
----------------------------------------------------------------------
diff --git a/content/installation-guide.html b/content/installation-guide.html
index ffb750e..2e7fab6 100644
--- a/content/installation-guide.html
+++ b/content/installation-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/lucene-datamap-guide.html
----------------------------------------------------------------------
diff --git a/content/lucene-datamap-guide.html b/content/lucene-datamap-guide.html
index 5f56429..7b1598e 100644
--- a/content/lucene-datamap-guide.html
+++ b/content/lucene-datamap-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
@@ -176,8 +158,9 @@
<div id="viewpage" name="viewpage">
<div class="row">
<div class="col-sm-12 col-md-12">
- <div><h1>
-<a id="carbondata-lucene-datamap-alpha-feature-in-140" class="anchor" href="#carbondata-lucene-datamap-alpha-feature-in-140" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Lucene DataMap (Alpha feature in 1.4.0)</h1>
+ <div>
+<h1>
+<a id="carbondata-lucene-datamap-alpha-feature" class="anchor" href="#carbondata-lucene-datamap-alpha-feature" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Lucene DataMap (Alpha Feature)</h1>
<ul>
<li><a href="#datamap-management">DataMap Management</a></li>
<li><a href="#lucene-datamap-introduction">Lucene Datamap</a></li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/mainpage.html
----------------------------------------------------------------------
diff --git a/content/mainpage.html b/content/mainpage.html
index 259d485..d515853 100644
--- a/content/mainpage.html
+++ b/content/mainpage.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
@@ -198,9 +180,11 @@
<li><a href="configuration-parameters.html">Configuring CarbonData</a></li>
<li><a href="streaming-guide.html">Streaming Guide</a></li>
<li><a href="sdk-guide.html">SDK Guide</a></li>
+ <li><a href="s3-guide.html">S3 Guide (Alpha Feature)</a></li>
<li><a href="datamap-developer-guide.html">DataMap Developer Guide</a></li>
- <li><a href="bloomfilter-datamap-guide.html">CarbonData BloomFilter DataMap (Alpha feature in 1.4.0)</a></li>
- <li><a href="lucene-datamap-guide.html">CarbonData Lucene DataMap (Alpha feature in 1.4.0)</a></li>
+ <li><a href="datamap-management.html">CarbonData DataMap Management</a></li>
+ <li><a href="bloomfilter-datamap-guide.html">CarbonData BloomFilter DataMap (Alpha Feature)</a></li>
+ <li><a href="lucene-datamap-guide.html">CarbonData Lucene DataMap (Alpha Feature)</a></li>
<li><a href="preaggregate-datamap-guide.html">CarbonData Pre-aggregate DataMap</a></li>
<li><a href="timeseries-datamap-guide.html">CarbonData Timeseries DataMap</a></li>
<li><a href="faq.html">FAQs</a></li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/pdf/CarbonData Documentation.pdf
----------------------------------------------------------------------
diff --git a/content/pdf/CarbonData Documentation.pdf b/content/pdf/CarbonData Documentation.pdf
index c4d389d..45ce585 100644
Binary files a/content/pdf/CarbonData Documentation.pdf and b/content/pdf/CarbonData Documentation.pdf differ
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/preaggregate-datamap-guide.html
----------------------------------------------------------------------
diff --git a/content/preaggregate-datamap-guide.html b/content/preaggregate-datamap-guide.html
index d195bfe..d68764d 100644
--- a/content/preaggregate-datamap-guide.html
+++ b/content/preaggregate-datamap-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
@@ -176,7 +158,8 @@
<div id="viewpage" name="viewpage">
<div class="row">
<div class="col-sm-12 col-md-12">
- <div><h1>
+ <div>
+<h1>
<a id="carbondata-pre-aggregate-datamap" class="anchor" href="#carbondata-pre-aggregate-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Pre-aggregate DataMap</h1>
<ul>
<li><a href="#quick-example">Quick Example</a></li>
@@ -422,6 +405,7 @@ change datatype command, CarbonData will check whether it will impact the pre-ag
not, the operation is allowed, otherwise operation will be rejected by throwing exception.</li>
<li>Partition management command: <code>ALTER TABLE ADD/DROP PARTITION</code>
</li>
+<li>Complex Datatypes for preaggregate is not supported.</li>
</ol>
<p>However, there is still way to support these operations on main table, in current CarbonData
release, user can do as following:</p>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/quick-start-guide.html
----------------------------------------------------------------------
diff --git a/content/quick-start-guide.html b/content/quick-start-guide.html
index 20cfbf1..89380b4 100644
--- a/content/quick-start-guide.html
+++ b/content/quick-start-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/s3-guide.html
----------------------------------------------------------------------
diff --git a/content/s3-guide.html b/content/s3-guide.html
new file mode 100644
index 0000000..d9edeaf
--- /dev/null
+++ b/content/s3-guide.html
@@ -0,0 +1,239 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'>
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <title>CarbonData</title>
+ <style>
+
+ </style>
+ <!-- Bootstrap -->
+
+ <link rel="stylesheet" href="css/bootstrap.min.css">
+ <link href="css/style.css" rel="stylesheet">
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="js/jquery.min.js"></script>
+ <script src="js/bootstrap.min.js"></script>
+
+
+</head>
+<body>
+<header>
+ <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper">
+ <div class="container">
+ <div class="navbar-header">
+ <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse"
+ class="navbar-toggle collapsed" type="button">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="index.html" class="logo">
+ <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/>
+ </a>
+ </div>
+ <div class="navbar-collapse collapse cd_navcontnt" id="navbar">
+ <ul class="nav navbar-nav navbar-right navlist-custom">
+ <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a>
+ </li>
+ <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false"> Download <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
+ target="_blank">Apache CarbonData 1.3.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
+ target="_blank">Apache CarbonData 1.3.0</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
+ target="_blank">Release Archive</a></li>
+ </ul>
+ </li>
+ <li><a href="mainpage.html" class="active">Documentation</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false">Community <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"
+ target="_blank">Contributing to CarbonData</a></li>
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"
+ target="_blank">Release Guide</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"
+ target="_blank">Project PMC and Committers</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"
+ target="_blank">CarbonData Meetups</a></li>
+ <li><a href="security.html">Apache CarbonData Security</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/CARBONDATA" target="_blank">Apache
+ Jira</a></li>
+ <li><a href="videogallery.html">CarbonData Videos </a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="apache_link hidden-xs dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="hidden-lg hidden-md hidden-sm dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li>
+ <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a>
+
+ </li>
+
+ </ul>
+ </div><!--/.nav-collapse -->
+ <div id="search-box">
+ <form method="get" action="http://www.google.com/search" target="_blank">
+ <div class="search-block">
+ <table border="0" cellpadding="0" width="100%">
+ <tr>
+ <td style="width:80%">
+ <input type="text" name="q" size=" 5" maxlength="255" value=""
+ class="search-input" placeholder="Search...." required/>
+ </td>
+ <td style="width:20%">
+ <input type="submit" value="Search"/></td>
+ </tr>
+ <tr>
+ <td align="left" style="font-size:75%" colspan="2">
+ <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/>
+ <span style=" position: relative; top: -3px;"> Only search for CarbonData</span>
+ </td>
+ </tr>
+ </table>
+ </div>
+ </form>
+ </div>
+ </div>
+ </nav>
+</header> <!-- end Header part -->
+
+<div class="fixed-padding"></div> <!-- top padding with fixde header -->
+
+<section><!-- Dashboard nav -->
+ <div class="container-fluid q">
+ <div class="col-sm-12 col-md-12 maindashboard">
+ <div class="row">
+ <section>
+ <div style="padding:10px 15px;">
+ <div id="viewpage" name="viewpage">
+ <div class="row">
+ <div class="col-sm-12 col-md-12">
+ <div>
+<p>#S3 Guide (Alpha Feature 1.4.1)</p>
+<p>Object storage is the recommended storage format in cloud as it can support storing large data
+files. S3 APIs are widely used for accessing object stores. This can be
+used to store or retrieve data on Amazon cloud, Huawei Cloud(OBS) or on any other object
+stores conforming to S3 API.
+Storing data in cloud is advantageous as there are no restrictions on the size of
+data and the data can be accessed from anywhere at any time.
+Carbondata can support any Object Storage that conforms to Amazon S3 API.
+Carbondata relies on Hadoop provided S3 filesystem APIs to access Object stores.</p>
+<p>#Writing to Object Storage</p>
+<p>To store carbondata files onto Object Store, <code>carbon.storelocation</code> property will have
+to be configured with Object Store path in CarbonProperties file.</p>
+<p>For example:</p>
+<pre><code>carbon.storelocation=s3a://mybucket/carbonstore.
+</code></pre>
+<p>If the existing store location cannot be changed or only specific tables need to be stored
+onto cloud object store, it can be done so by specifying the <code>location</code> option in the create
+table DDL command.</p>
+<p>For example:</p>
+<pre><code>CREATE TABLE IF NOT EXISTS db1.table1(col1 string, col2 int) STORED AS carbondata LOCATION 's3a://mybucket/carbonstore'
+</code></pre>
+<p>For more details on create table, Refer <a href="./data-management-on-carbondata.html#create-table">data-management-on-carbondata</a></p>
+<p>#Authentication</p>
+<p>Authentication properties will have to be configured to store the carbondata files on to S3 location.</p>
+<p>Authentication properties can be set in any of the following ways:</p>
+<ol>
+<li>
+<p>Set authentication properties in core-site.xml, refer
+<a href="https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html#Authentication_properties" rel="nofollow">hadoop authentication document</a></p>
+</li>
+<li>
+<p>Set authentication properties in spark-defaults.conf.</p>
+</li>
+</ol>
+<p>Example</p>
+<pre><code>spark.hadoop.fs.s3a.secret.key=123
+spark.hadoop.fs.s3a.access.key=456
+</code></pre>
+<ol start="3">
+<li>Pass authentication properties with spark-submit as configuration.</li>
+</ol>
+<p>Example:</p>
+<pre><code>./bin/spark-submit --master yarn --conf spark.hadoop.fs.s3a.secret.key=123 --conf spark.hadoop.fs
+.s3a.access.key=456 --class=
+</code></pre>
+<ol start="4">
+<li>Set authentication properties to hadoop configuration object in sparkContext.</li>
+</ol>
+<p>Example:</p>
+<pre><code>sparkSession.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", "123")
+sparkSession.sparkContext.hadoopConfiguration.set("fs.s3a.access.key","456")
+</code></pre>
+<p>#Recommendations</p>
+<ol>
+<li>Object Storage like S3 does not support file leasing mechanism(supported by HDFS) that is
+required to take locks which ensure consistency between concurrent operations therefore, it is
+recommended to set the configurable lock path property(<a href="https://github.com/apache/carbondata/blob/master/docs/configuration-parameters.html#miscellaneous-configuration">carbon.lock.path</a>)
+to a HDFS directory.</li>
+<li>Concurrent data manipulation operations are not supported. Object stores follow eventual
+consistency semantics, i.e., any put request might take some time to reflect when trying to list
+.This behaviour causes not to ensure the data read is always consistent or latest.</li>
+</ol>
+</div>
+</div>
+</div>
+</div>
+<div class="doc-footer">
+ <a href="#top" class="scroll-top">Top</a>
+</div>
+</div>
+</section>
+</div>
+</div>
+</div>
+</section><!-- End systemblock part -->
+<script src="js/custom.js"></script>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/sdk-guide.html
----------------------------------------------------------------------
diff --git a/content/sdk-guide.html b/content/sdk-guide.html
index d980f83..cc55dc0 100644
--- a/content/sdk-guide.html
+++ b/content/sdk-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
@@ -176,7 +158,8 @@
<div id="viewpage" name="viewpage">
<div class="row">
<div class="col-sm-12 col-md-12">
- <div><h1>
+ <div>
+<h1>
<a id="sdk-guide" class="anchor" href="#sdk-guide" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>SDK Guide</h1>
<p>In the carbon jars package, there exist a carbondata-store-sdk-x.x.x-SNAPSHOT.jar, including SDK writer and reader.</p>
<h1>
@@ -286,6 +269,47 @@ These SDK writer output contains just a carbondata and carbonindex files. No met
}
}
}</pre></div>
+<h3>
+<a id="example-with-json-format" class="anchor" href="#example-with-json-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example with Json format</h3>
+<div class="highlight highlight-source-java"><pre><span class="pl-k">import</span> <span class="pl-smi">java.io.IOException</span>;
+
+<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException</span>;
+<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.core.metadata.datatype.DataTypes</span>;
+<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.core.util.CarbonProperties</span>;
+<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.CarbonWriter</span>;
+<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.CarbonWriterBuilder</span>;
+<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.Field</span>;
+<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.Schema</span>;
+
+<span class="pl-k">public</span> <span class="pl-k">class</span> <span class="pl-en">TestSdkJson</span> {
+
+ <span class="pl-k">public</span> <span class="pl-k">static</span> <span class="pl-k">void</span> <span class="pl-en">main</span>(<span class="pl-k">String</span>[] <span class="pl-v">args</span>) <span class="pl-k">throws</span> <span class="pl-smi">InvalidLoadOptionException</span> {
+ testJsonSdkWriter();
+ }
+
+ <span class="pl-k">public</span> <span class="pl-k">static</span> <span class="pl-k">void</span> <span class="pl-en">testJsonSdkWriter</span>() <span class="pl-k">throws</span> <span class="pl-smi">InvalidLoadOptionException</span> {
+ <span class="pl-smi">String</span> path <span class="pl-k">=</span> <span class="pl-s"><span class="pl-pds">"</span>./target/testJsonSdkWriter<span class="pl-pds">"</span></span>;
+
+ <span class="pl-k">Field</span>[] fields <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Field</span>[<span class="pl-c1">2</span>];
+ fields[<span class="pl-c1">0</span>] <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Field</span>(<span class="pl-s"><span class="pl-pds">"</span>name<span class="pl-pds">"</span></span>, <span class="pl-smi">DataTypes</span><span class="pl-c1"><span class="pl-k">.</span>STRING</span>);
+ fields[<span class="pl-c1">1</span>] <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Field</span>(<span class="pl-s"><span class="pl-pds">"</span>age<span class="pl-pds">"</span></span>, <span class="pl-smi">DataTypes</span><span class="pl-c1"><span class="pl-k">.</span>INT</span>);
+
+ <span class="pl-smi">Schema</span> <span class="pl-smi">CarbonSchema</span> <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Schema</span>(fields);
+
+ <span class="pl-smi">CarbonWriterBuilder</span> builder <span class="pl-k">=</span> <span class="pl-smi">CarbonWriter</span><span class="pl-k">.</span>builder()<span class="pl-k">.</span>outputPath(path);
+
+ <span class="pl-c"><span class="pl-c">//</span> initialize json writer with carbon schema</span>
+ <span class="pl-smi">CarbonWriter</span> writer <span class="pl-k">=</span> builder<span class="pl-k">.</span>buildWriterForJsonInput(<span class="pl-smi">CarbonSchema</span>);
+ <span class="pl-c"><span class="pl-c">//</span> one row of json Data as String</span>
+ <span class="pl-smi">String</span> <span class="pl-smi">JsonRow</span> <span class="pl-k">=</span> <span class="pl-s"><span class="pl-pds">"</span>{<span class="pl-cce">\"</span>name<span class="pl-cce">\"</span>:<span class="pl-cce">\"</span>abcd<span class="pl-cce">\"</span>, <span class="pl-cce">\"</span>age<span class="pl-cce">\"</span>:10}<span class="pl-pds">"</span></span>;
+
+ <span class="pl-k">int</span> rows <span class="pl-k">=</span> <span class="pl-c1">5</span>;
+ <span class="pl-k">for</span> (<span class="pl-k">int</span> i <span class="pl-k">=</span> <span class="pl-c1">0</span>; i <span class="pl-k"><</span> rows; i<span class="pl-k">++</span>) {
+ writer<span class="pl-k">.</span>write(<span class="pl-smi">JsonRow</span>);
+ }
+ writer<span class="pl-k">.</span>close();
+ }
+} </pre></div>
<h2>
<a id="datatypes-mapping" class="anchor" href="#datatypes-mapping" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Datatypes Mapping</h2>
<p>Each of SQL data types are mapped into data types of SDK. Following are the mapping:</p>
@@ -339,6 +363,20 @@ These SDK writer output contains just a carbondata and carbonindex files. No met
</tr>
</tbody>
</table>
+<p><strong>NOTE:</strong>
+Carbon Supports below logical types of AVRO.
+a. Date
+The date logical type represents a date within the calendar, with no reference to a particular time zone or time of day.
+A date logical type annotates an Avro int, where the int stores the number of days from the unix epoch, 1 January 1970 (ISO calendar).
+b. Timestamp (millisecond precision)
+The timestamp-millis logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one millisecond.
+A timestamp-millis logical type annotates an Avro long, where the long stores the number of milliseconds from the unix epoch, 1 January 1970 00:00:00.000 UTC.
+c. Timestamp (microsecond precision)
+The timestamp-micros logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one microsecond.
+A timestamp-micros logical type annotates an Avro long, where the long stores the number of microseconds from the unix epoch, 1 January 1970 00:00:00.000000 UTC.</p>
+<pre><code>Currently the values of logical types are not validated by carbon.
+Expect that avro record passed by the user is already validated by avro record generator tools.
+</code></pre>
<h2>
<a id="run-sql-on-files-directly" class="anchor" href="#run-sql-on-files-directly" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Run SQL on files directly</h2>
<p>Instead of creating table and query it, you can also query that file directly with SQL.</p>
@@ -396,6 +434,18 @@ public CarbonWriterBuilder withBlockSize(int blockSize);
public CarbonWriterBuilder withBlockletSize(int blockletSize);
</code></pre>
<pre><code>/**
+ * @param enableLocalDictionary enable local dictionary , default is false
+ * @return updated CarbonWriterBuilder
+ */
+public CarbonWriterBuilder enableLocalDictionary(boolean enableLocalDictionary);
+</code></pre>
+<pre><code>/**
+ * @param localDictionaryThreshold is localDictionaryThreshold,default is 10000
+ * @return updated CarbonWriterBuilder
+ */
+public CarbonWriterBuilder localDictionaryThreshold(int localDictionaryThreshold) ;
+</code></pre>
+<pre><code>/**
* sets the list of columns that needs to be in sorted order
* @param sortColumns is a string array of columns that needs to be sorted.
* If it is null or by default all dimensions are selected for sorting
@@ -419,7 +469,7 @@ public CarbonWriterBuilder persistSchemaFile(boolean persist);
* by default it is system time in nano seconds.
* @return updated CarbonWriterBuilder
*/
-public CarbonWriterBuilder taskNo(String taskNo);
+public CarbonWriterBuilder taskNo(long taskNo);
</code></pre>
<pre><code>/**
* To support the load options for sdk writer
@@ -458,7 +508,7 @@ public CarbonWriterBuilder withLoadOptions(Map<String, String> options);
* @throws IOException
* @throws InvalidLoadOptionException
*/
-public CarbonWriter buildWriterForCSVInput() throws IOException, InvalidLoadOptionException;
+public CarbonWriter buildWriterForCSVInput(org.apache.carbondata.sdk.file.Schema schema) throws IOException, InvalidLoadOptionException;
</code></pre>
<pre><code>/**
* Build a {@link CarbonWriter}, which accepts Avro format object
@@ -467,14 +517,25 @@ public CarbonWriter buildWriterForCSVInput() throws IOException, InvalidLoadOpti
* @throws IOException
* @throws InvalidLoadOptionException
*/
-public CarbonWriter buildWriterForAvroInput() throws IOException, InvalidLoadOptionException;
+public CarbonWriter buildWriterForAvroInput(org.apache.avro.Schema schema) throws IOException, InvalidLoadOptionException;
+</code></pre>
+<pre><code>/**
+* Build a {@link CarbonWriter}, which accepts Json object
+* @param carbonSchema carbon Schema object
+* @return JsonCarbonWriter
+* @throws IOException
+* @throws InvalidLoadOptionException
+*/
+public JsonCarbonWriter buildWriterForJsonInput(Schema carbonSchema);
</code></pre>
<h3>
<a id="class-orgapachecarbondatasdkfilecarbonwriter" class="anchor" href="#class-orgapachecarbondatasdkfilecarbonwriter" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.CarbonWriter</h3>
<pre><code>/**
* Write an object to the file, the format of the object depends on the implementation
-* If AvroCarbonWriter, object is of type org.apache.avro.generic.GenericData.Record
-* If CSVCarbonWriter, object is of type String[]
+* If AvroCarbonWriter, object is of type org.apache.avro.generic.GenericData.Record,
+* which is one row of data.
+* If CSVCarbonWriter, object is of type String[], which is one row of data
+* If JsonCarbonWriter, object is of type String, which is one row of json
* Note: This API is not thread safe
* @param object
* @throws IOException
@@ -624,14 +685,6 @@ External client can make use of this reader to read CarbonData files without Car
public CarbonReaderBuilder projection(String[] projectionColumnNames);
</code></pre>
<pre><code> /**
- * Project all Columns for carbon reader
- *
- * @return CarbonReaderBuilder object
- * @throws IOException
- */
- public CarbonReaderBuilder projectAllColumns();
-</code></pre>
-<pre><code> /**
* Configure the transactional status of table
* If set to false, then reads the carbondata and carbonindex files from a flat folder structure.
* If set to true, then reads the carbondata and carbonindex files from segment folder structure.
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/streaming-guide.html
----------------------------------------------------------------------
diff --git a/content/streaming-guide.html b/content/streaming-guide.html
index 981a095..6e2e7ee 100644
--- a/content/streaming-guide.html
+++ b/content/streaming-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
@@ -176,7 +158,8 @@
<div id="viewpage" name="viewpage">
<div class="row">
<div class="col-sm-12 col-md-12">
- <div><h1>
+ <div>
+<h1>
<a id="carbondata-streaming-ingestion" class="anchor" href="#carbondata-streaming-ingestion" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Streaming Ingestion</h1>
<h2>
<a id="quick-example" class="anchor" href="#quick-example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Quick example</h2>
@@ -198,6 +181,7 @@
<span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">spark</span>.<span class="pl-en">sql</span>.<span class="pl-en">CarbonSession</span>.<span class="pl-en">_</span>
<span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">spark</span>.<span class="pl-en">sql</span>.<span class="pl-en">streaming</span>.{<span class="pl-en">ProcessingTime</span>, <span class="pl-en">StreamingQuery</span>}
<span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">carbondata</span>.<span class="pl-en">core</span>.<span class="pl-en">util</span>.<span class="pl-en">path</span>.<span class="pl-en">CarbonTablePath</span>
+ <span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">carbondata</span>.<span class="pl-en">streaming</span>.<span class="pl-en">parser</span>.<span class="pl-en">CarbonStreamParser</span>
<span class="pl-k">val</span> <span class="pl-smi">warehouse</span> <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-en">File</span>(<span class="pl-s"><span class="pl-pds">"</span>./warehouse<span class="pl-pds">"</span></span>).getCanonicalPath
<span class="pl-k">val</span> <span class="pl-smi">metastore</span> <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-en">File</span>(<span class="pl-s"><span class="pl-pds">"</span>./metastore<span class="pl-pds">"</span></span>).getCanonicalPath
@@ -241,6 +225,8 @@
.option(<span class="pl-s"><span class="pl-pds">"</span>checkpointLocation<span class="pl-pds">"</span></span>, <span class="pl-en">CarbonTablePath</span>.getStreamingCheckpointDir(tablePath))
.option(<span class="pl-s"><span class="pl-pds">"</span>dbName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>default<span class="pl-pds">"</span></span>)
.option(<span class="pl-s"><span class="pl-pds">"</span>tableName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>carbon_table<span class="pl-pds">"</span></span>)
+ .option(<span class="pl-en">CarbonStreamParser</span>.<span class="pl-en">CARBON_STREAM_PARSER</span>,
+ <span class="pl-en">CarbonStreamParser</span>.<span class="pl-en">CARBON_STREAM_PARSER_CSV</span>)
.start()
<span class="pl-c"><span class="pl-c">//</span> start new thread to show data</span>
@@ -370,14 +356,14 @@ streaming table using following DDL.</p>
<tbody>
<tr>
<td>carbon.stream.parser</td>
-<td>org.apache.carbondata.streaming.parser.CSVStreamParserImp</td>
+<td>org.apache.carbondata.streaming.parser.RowStreamParserImp</td>
<td>the class of the stream parser</td>
</tr>
</tbody>
</table>
<p>Currently CarbonData support two parsers, as following:</p>
-<p><strong>1. org.apache.carbondata.streaming.parser.CSVStreamParserImp</strong>: This is the default stream parser, it gets a line data(String type) from the first index of InternalRow and converts this String to Object[].</p>
-<p><strong>2. org.apache.carbondata.streaming.parser.RowStreamParserImp</strong>: This stream parser will auto convert InternalRow to Object[] according to schema of this <code>DataSet</code>, for example:</p>
+<p><strong>1. org.apache.carbondata.streaming.parser.CSVStreamParserImp</strong>: This parser gets a line data(String type) from the first index of InternalRow and converts this String to Object[].</p>
+<p><strong>2. org.apache.carbondata.streaming.parser.RowStreamParserImp</strong>: This is the default stream parser, it will auto convert InternalRow to Object[] according to schema of this <code>DataSet</code>, for example:</p>
<div class="highlight highlight-source-scala"><pre> <span class="pl-k">case</span> <span class="pl-k">class</span> <span class="pl-en">FileElement</span>(<span class="pl-v">school</span>: <span class="pl-en">Array</span>[<span class="pl-k">String</span>], <span class="pl-v">age</span>: <span class="pl-k">Int</span>)
<span class="pl-k">case</span> <span class="pl-k">class</span> <span class="pl-en">StreamData</span>(<span class="pl-v">id</span>: <span class="pl-k">Int</span>, <span class="pl-v">name</span>: <span class="pl-k">String</span>, <span class="pl-v">city</span>: <span class="pl-k">String</span>, <span class="pl-v">salary</span>: <span class="pl-k">Float</span>, <span class="pl-v">file</span>: <span class="pl-en">FileElement</span>)
...
@@ -403,8 +389,6 @@ streaming table using following DDL.</p>
.option(<span class="pl-s"><span class="pl-pds">"</span>checkpointLocation<span class="pl-pds">"</span></span>, tablePath.getStreamingCheckpointDir)
.option(<span class="pl-s"><span class="pl-pds">"</span>dbName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>default<span class="pl-pds">"</span></span>)
.option(<span class="pl-s"><span class="pl-pds">"</span>tableName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>carbon_table<span class="pl-pds">"</span></span>)
- .option(<span class="pl-en">CarbonStreamParser</span>.<span class="pl-en">CARBON_STREAM_PARSER</span>,
- <span class="pl-en">CarbonStreamParser</span>.<span class="pl-en">CARBON_STREAM_PARSER_ROW_PARSER</span>)
.start()
...</pre></div>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/supported-data-types-in-carbondata.html
----------------------------------------------------------------------
diff --git a/content/supported-data-types-in-carbondata.html b/content/supported-data-types-in-carbondata.html
index 0f06a42..780cab0 100644
--- a/content/supported-data-types-in-carbondata.html
+++ b/content/supported-data-types-in-carbondata.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
@@ -206,6 +188,8 @@
<li>CHAR</li>
<li>VARCHAR</li>
</ul>
+<p><strong>NOTE</strong>: For string longer than 32000 characters, use <code>LONG_STRING_COLUMNS</code> in table property.
+Please refer to TBLProperties in <a href="https://github.com/apache/carbondata/blob/master/docs/data-management-on-carbondata.html#create-table">CreateTable</a> for more information.</p>
</li>
<li>
<p>Complex Types</p>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/timeseries-datamap-guide.html
----------------------------------------------------------------------
diff --git a/content/timeseries-datamap-guide.html b/content/timeseries-datamap-guide.html
index 2400ce0..548be2e 100644
--- a/content/timeseries-datamap-guide.html
+++ b/content/timeseries-datamap-guide.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
@@ -176,7 +158,8 @@
<div id="viewpage" name="viewpage">
<div class="row">
<div class="col-sm-12 col-md-12">
- <div><h1>
+ <div>
+<h1>
<a id="carbondata-timeseries-datamap" class="anchor" href="#carbondata-timeseries-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Timeseries DataMap</h1>
<ul>
<li><a href="#timeseries-datamap-intoduction">Timeseries DataMap Introduction</a></li>
@@ -184,7 +167,7 @@
<li><a href="#data-management-with-pre-aggregate-tables">Data Management</a></li>
</ul>
<h2>
-<a id="timeseries-datamap-introduction-alpha-feature-in-130" class="anchor" href="#timeseries-datamap-introduction-alpha-feature-in-130" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Timeseries DataMap Introduction (Alpha feature in 1.3.0)</h2>
+<a id="timeseries-datamap-introduction-alpha-feature" class="anchor" href="#timeseries-datamap-introduction-alpha-feature" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Timeseries DataMap Introduction (Alpha Feature)</h2>
<p>Timeseries DataMap a pre-aggregate table implementation based on 'pre-aggregate' DataMap.
Difference is that Timeseries DataMap has built-in understanding of time hierarchy and
levels: year, month, day, hour, minute, so that it supports automatic roll-up in time dimension
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/troubleshooting.html
----------------------------------------------------------------------
diff --git a/content/troubleshooting.html b/content/troubleshooting.html
index d5cf1b2..c668dc9 100644
--- a/content/troubleshooting.html
+++ b/content/troubleshooting.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/useful-tips-on-carbondata.html
----------------------------------------------------------------------
diff --git a/content/useful-tips-on-carbondata.html b/content/useful-tips-on-carbondata.html
index e8b0ab0..912cb48 100644
--- a/content/useful-tips-on-carbondata.html
+++ b/content/useful-tips-on-carbondata.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
@@ -180,7 +162,7 @@
<h1>
<a id="useful-tips" class="anchor" href="#useful-tips" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Useful Tips</h1>
<p>This tutorial guides you to create CarbonData Tables and optimize performance.
-The following sections will elaborate on the above topics :</p>
+The following sections will elaborate on the below topics :</p>
<ul>
<li><a href="#suggestions-to-create-carbondata-table">Suggestions to create CarbonData Table</a></li>
<li><a href="#configuration-for-optimizing-data-loading-performance-for-massive-data">Configuration for Optimizing Data Loading performance for Massive Data</a></li>
@@ -326,6 +308,10 @@ Incremental values are efficient in using min/max index. The create table comman
)STORED BY 'carbondata'
TBLPROPERTIES ('SORT_COLUMNS'='Dime_1, HOST, MSISDN')
</code></pre>
+<p><strong>NOTE:</strong></p>
+<ul>
+<li>BloomFilter can be created to enhance performance for queries with precise equal/in conditions. You can find more information about it in BloomFilter datamap <a href="https://github.com/apache/carbondata/blob/master/docs/datamap/bloomfilter-datamap-guide.html" target=_blank>document</a>.</li>
+</ul>
<h2>
<a id="configuration-for-optimizing-data-loading-performance-for-massive-data" class="anchor" href="#configuration-for-optimizing-data-loading-performance-for-massive-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Configuration for Optimizing Data Loading performance for Massive Data</h2>
<p>CarbonData supports large data load, in this process sorting data while loading consumes a lot of memory and disk IO and
@@ -457,7 +443,7 @@ scenarios. After the completion of POC, some of the configurations impacting the
<td>spark/carbonlib/carbon.properties</td>
<td>Data loading</td>
<td>Specify the name of compressor to compress the intermediate sort temporary files during sort procedure in data loading.</td>
-<td>The optional values are 'SNAPPY','GZIP','BZIP2','LZ4' and empty. By default, empty means that Carbondata will not compress the sort temp files. This parameter will be useful if you encounter disk bottleneck.</td>
+<td>The optional values are 'SNAPPY','GZIP','BZIP2','LZ4','ZSTD' and empty. By default, empty means that Carbondata will not compress the sort temp files. This parameter will be useful if you encounter disk bottleneck.</td>
</tr>
<tr>
<td>carbon.load.skewedDataOptimization.enabled</td>
@@ -466,6 +452,13 @@ scenarios. After the completion of POC, some of the configurations impacting the
<td>Whether to enable size based block allocation strategy for data loading.</td>
<td>When loading, carbondata will use file size based block allocation strategy for task distribution. It will make sure that all the executors process the same size of data -- It's useful if the size of your input data files varies widely, say 1MB~1GB.</td>
</tr>
+<tr>
+<td>carbon.load.min.size.enabled</td>
+<td>spark/carbonlib/carbon.properties</td>
+<td>Data loading</td>
+<td>Whether to enable node minumun input data size allocation strategy for data loading.</td>
+<td>When loading, carbondata will use node minumun input data size allocation strategy for task distribution. It will make sure the node load the minimum amount of data -- It's useful if the size of your input data files very small, say 1MB~256MB,Avoid generating a large number of small files.</td>
+</tr>
</tbody>
</table>
<p>Note: If your CarbonData instance is provided only for query, you may specify the property 'spark.speculation=true' which is in conf directory of spark.</p>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index df5280b..ba5b4b5 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -9,13 +9,15 @@ fileList=["configuration-parameters",
"troubleshooting",
"useful-tips-on-carbondata",
"sdk-guide",
+ "s3-guide",
"datamap-developer-guide"
]
dataMapFileList=[
"bloomfilter-datamap-guide",
"lucene-datamap-guide",
"preaggregate-datamap-guide",
- "timeseries-datamap-guide"]
+ "timeseries-datamap-guide",
+ "datamap-management"]
dataMapFilesUrl="https://raw.githubusercontent.com/apache/carbondata/master/docs/datamap/"
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/src/main/scala/html/header.html
----------------------------------------------------------------------
diff --git a/src/main/scala/html/header.html b/src/main/scala/html/header.html
index 470de01..9e904d0 100644
--- a/src/main/scala/html/header.html
+++ b/src/main/scala/html/header.html
@@ -51,6 +51,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
target="_blank">Apache CarbonData 1.4.0</a></li>
<li>
@@ -60,27 +63,6 @@
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
target="_blank">Apache CarbonData 1.3.0</a></li>
<li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>