You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2018/10/17 10:15:03 UTC
[15/20] carbondata-site git commit: modified for 1.5.0 version
modified for 1.5.0 version
Project: http://git-wip-us.apache.org/repos/asf/carbondata-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata-site/commit/6ad7599a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata-site/tree/6ad7599a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata-site/diff/6ad7599a
Branch: refs/heads/asf-site
Commit: 6ad7599a8cc7f9a577247a563e9a24038c0de7fc
Parents: bee5633
Author: Raghunandan S <ca...@gmail.com>
Authored: Tue Oct 9 10:03:04 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Wed Oct 17 06:35:21 2018 +0530
----------------------------------------------------------------------
content/CSDK-guide.html | 422 ++++++
content/WEB-INF/classes/application.conf | 4 +-
content/bloomfilter-datamap-guide.html | 16 +-
content/carbon-as-spark-datasource-guide.html | 349 +++++
content/configuration-parameters.html | 187 +--
content/css/style.css | 5 +
content/data-management-on-carbondata.html | 1321 ------------------
content/data-management.html | 413 ------
content/datamap-developer-guide.html | 14 +-
content/datamap-management.html | 18 +-
content/ddl-of-carbondata.html | 210 ++-
content/ddl-operation-on-carbondata.html | 748 ----------
content/dml-of-carbondata.html | 26 +-
content/dml-operation-on-carbondata.html | 716 ----------
content/documentation.html | 12 +-
content/faq.html | 31 +-
content/file-structure-of-carbondata.html | 12 +-
.../how-to-contribute-to-apache-carbondata.html | 14 +-
content/index.html | 12 +-
content/installation-guide.html | 455 ------
content/introduction.html | 30 +-
content/language-manual.html | 13 +-
content/lucene-datamap-guide.html | 14 +-
content/mainpage.html | 214 ---
content/performance-tuning.html | 20 +-
content/preaggregate-datamap-guide.html | 12 +-
content/quick-start-guide.html | 45 +-
content/release-guide.html | 10 +-
content/s3-guide.html | 12 +-
content/sdk-guide.html | 323 +++--
content/sdk-writer-guide.html | 549 --------
content/security.html | 2 +-
content/segment-management-on-carbondata.html | 10 +-
content/streaming-guide.html | 49 +-
content/supported-data-types-in-carbondata.html | 15 +-
content/timeseries-datamap-guide.html | 10 +-
content/troubleshooting.html | 366 -----
content/usecases.html | 42 +-
content/useful-tips-on-carbondata.html | 480 -------
src/main/resources/application.conf | 4 +-
src/main/scala/MdFileHandler.scala | 2 +-
src/main/scala/html/header.html | 10 +-
src/main/scala/scripts/CSDK-guide | 11 +
.../scripts/carbon-as-spark-datasource-guide | 11 +
src/main/scala/scripts/sdk-guide | 11 +-
src/main/webapp/CSDK-guide.html | 422 ++++++
src/main/webapp/bloomfilter-datamap-guide.html | 16 +-
.../carbon-as-spark-datasource-guide.html | 349 +++++
src/main/webapp/configuration-parameters.html | 187 +--
src/main/webapp/css/style.css | 5 +
src/main/webapp/datamap-developer-guide.html | 14 +-
src/main/webapp/datamap-management.html | 18 +-
src/main/webapp/ddl-of-carbondata.html | 210 ++-
src/main/webapp/dml-of-carbondata.html | 26 +-
src/main/webapp/documentation.html | 12 +-
src/main/webapp/faq.html | 31 +-
.../webapp/file-structure-of-carbondata.html | 12 +-
.../how-to-contribute-to-apache-carbondata.html | 14 +-
src/main/webapp/introduction.html | 30 +-
src/main/webapp/language-manual.html | 13 +-
src/main/webapp/lucene-datamap-guide.html | 14 +-
src/main/webapp/performance-tuning.html | 20 +-
src/main/webapp/preaggregate-datamap-guide.html | 12 +-
src/main/webapp/quick-start-guide.html | 45 +-
src/main/webapp/release-guide.html | 10 +-
src/main/webapp/s3-guide.html | 12 +-
src/main/webapp/sdk-guide.html | 323 +++--
.../segment-management-on-carbondata.html | 10 +-
src/main/webapp/streaming-guide.html | 49 +-
.../supported-data-types-in-carbondata.html | 15 +-
src/main/webapp/timeseries-datamap-guide.html | 10 +-
src/main/webapp/usecases.html | 42 +-
src/site/markdown/CSDK-guide.md | 197 +++
src/site/markdown/bloomfilter-datamap-guide.md | 6 +-
.../carbon-as-spark-datasource-guide.md | 99 ++
src/site/markdown/configuration-parameters.md | 163 +--
src/site/markdown/datamap-developer-guide.md | 4 +-
src/site/markdown/datamap-management.md | 8 +-
src/site/markdown/ddl-of-carbondata.md | 167 ++-
src/site/markdown/dml-of-carbondata.md | 20 +-
src/site/markdown/documentation.md | 2 +-
src/site/markdown/faq.md | 22 +-
.../markdown/file-structure-of-carbondata.md | 2 +-
.../how-to-contribute-to-apache-carbondata.md | 4 +-
src/site/markdown/introduction.md | 20 +-
src/site/markdown/language-manual.md | 4 +-
src/site/markdown/lucene-datamap-guide.md | 4 +-
src/site/markdown/performance-tuning.md | 10 +-
src/site/markdown/preaggregate-datamap-guide.md | 2 +-
src/site/markdown/quick-start-guide.md | 35 +-
src/site/markdown/s3-guide.md | 2 +-
src/site/markdown/sdk-guide.md | 281 ++--
src/site/markdown/streaming-guide.md | 39 +-
.../supported-data-types-in-carbondata.md | 5 +
src/site/markdown/usecases.md | 32 +-
95 files changed, 3809 insertions(+), 6490 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/CSDK-guide.html
----------------------------------------------------------------------
diff --git a/content/CSDK-guide.html b/content/CSDK-guide.html
new file mode 100644
index 0000000..193d74f
--- /dev/null
+++ b/content/CSDK-guide.html
@@ -0,0 +1,422 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'>
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <title>CarbonData</title>
+ <style>
+
+ </style>
+ <!-- Bootstrap -->
+
+ <link rel="stylesheet" href="css/bootstrap.min.css">
+ <link href="css/style.css" rel="stylesheet">
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="js/jquery.min.js"></script>
+ <script src="js/bootstrap.min.js"></script>
+ <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js"></script>
+
+
+</head>
+<body>
+<header>
+ <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper">
+ <div class="container">
+ <div class="navbar-header">
+ <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse"
+ class="navbar-toggle collapsed" type="button">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="index.html" class="logo">
+ <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/>
+ </a>
+ </div>
+ <div class="navbar-collapse collapse cd_navcontnt" id="navbar">
+ <ul class="nav navbar-nav navbar-right navlist-custom">
+ <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a>
+ </li>
+ <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false"> Download <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
+ target="_blank">Apache CarbonData 1.3.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
+ target="_blank">Apache CarbonData 1.3.0</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
+ target="_blank">Release Archive</a></li>
+ </ul>
+ </li>
+ <li><a href="documentation.html" class="active">Documentation</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false">Community <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"
+ target="_blank">Contributing to CarbonData</a></li>
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"
+ target="_blank">Release Guide</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"
+ target="_blank">Project PMC and Committers</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"
+ target="_blank">CarbonData Meetups</a></li>
+ <li><a href="security.html">Apache CarbonData Security</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/CARBONDATA" target="_blank">Apache
+ Jira</a></li>
+ <li><a href="videogallery.html">CarbonData Videos </a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="apache_link hidden-xs dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="hidden-lg hidden-md hidden-sm dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li>
+ <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a>
+
+ </li>
+
+ </ul>
+ </div><!--/.nav-collapse -->
+ <div id="search-box">
+ <form method="get" action="http://www.google.com/search" target="_blank">
+ <div class="search-block">
+ <table border="0" cellpadding="0" width="100%">
+ <tr>
+ <td style="width:80%">
+ <input type="text" name="q" size=" 5" maxlength="255" value=""
+ class="search-input" placeholder="Search...." required/>
+ </td>
+ <td style="width:20%">
+ <input type="submit" value="Search"/></td>
+ </tr>
+ <tr>
+ <td align="left" style="font-size:75%" colspan="2">
+ <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/>
+ <span style=" position: relative; top: -3px;"> Only search for CarbonData</span>
+ </td>
+ </tr>
+ </table>
+ </div>
+ </form>
+ </div>
+ </div>
+ </nav>
+</header> <!-- end Header part -->
+
+<div class="fixed-padding"></div> <!-- top padding with fixde header -->
+
+<section><!-- Dashboard nav -->
+ <div class="container-fluid q">
+ <div class="col-sm-12 col-md-12 maindashboard">
+ <div class="verticalnavbar">
+ <nav class="b-sticky-nav">
+ <div class="nav-scroller">
+ <div class="nav__inner">
+ <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a>
+ <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a>
+ <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a>
+ <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a>
+ <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a>
+ <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a>
+ <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a>
+ <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a>
+ <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a>
+ </div>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a>
+ <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a>
+ <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
+ <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
+ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ </div>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
+ <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
+ <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
+ <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
+ <a class="b-nav__security nav__item" href="./security.html">Security</a>
+ <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a>
+ </div>
+ </div>
+ <div class="navindicator">
+ <div class="b-nav__intro navindicator__item"></div>
+ <div class="b-nav__quickstart navindicator__item"></div>
+ <div class="b-nav__uses navindicator__item"></div>
+ <div class="b-nav__docs navindicator__item"></div>
+ <div class="b-nav__datamap navindicator__item"></div>
+ <div class="b-nav__api navindicator__item"></div>
+ <div class="b-nav__perf navindicator__item"></div>
+ <div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__faq navindicator__item"></div>
+ <div class="b-nav__contri navindicator__item"></div>
+ <div class="b-nav__security navindicator__item"></div>
+ </div>
+ </nav>
+ </div>
+ <div class="mdcontent">
+ <section>
+ <div style="padding:10px 15px;">
+ <div id="viewpage" name="viewpage">
+ <div class="row">
+ <div class="col-sm-12 col-md-12">
+ <div>
+<h1>
+<a id="csdk-guide" class="anchor" href="#csdk-guide" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CSDK Guide</h1>
+<p>CarbonData CSDK provides C++ interface to write and read carbon file.
+CSDK use JNI to invoke java SDK in C++ code.</p>
+<h1>
+<a id="csdk-reader" class="anchor" href="#csdk-reader" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CSDK Reader</h1>
+<p>This CSDK reader reads CarbonData file and carbonindex file at a given path.
+External client can make use of this reader to read CarbonData files in C++
+code and without CarbonSession.</p>
+<p>In the carbon jars package, there exist a carbondata-sdk.jar,
+including SDK reader for CSDK.</p>
+<h2>
+<a id="quick-example" class="anchor" href="#quick-example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Quick example</h2>
+<pre><code>// 1. init JVM
+JavaVM *jvm;
+JNIEnv *initJVM() {
+ JNIEnv *env;
+ JavaVMInitArgs vm_args;
+ int parNum = 3;
+ int res;
+ JavaVMOption options[parNum];
+
+ options[0].optionString = "-Djava.compiler=NONE";
+ options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar";
+ options[2].optionString = "-verbose:jni";
+ vm_args.version = JNI_VERSION_1_8;
+ vm_args.nOptions = parNum;
+ vm_args.options = options;
+ vm_args.ignoreUnrecognized = JNI_FALSE;
+
+ res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args);
+ if (res < 0) {
+ fprintf(stderr, "\nCan't create Java VM\n");
+ exit(1);
+ }
+
+ return env;
+}
+
+// 2. create carbon reader and read data
+// 2.1 read data from local disk
+/**
+ * test read data from local disk, without projection
+ *
+ * @param env jni env
+ * @return
+ */
+bool readFromLocalWithoutProjection(JNIEnv *env) {
+
+ CarbonReader carbonReaderClass;
+ carbonReaderClass.builder(env, "../resources/carbondata", "test");
+ carbonReaderClass.build();
+
+ while (carbonReaderClass.hasNext()) {
+ jobjectArray row = carbonReaderClass.readNextRow();
+ jsize length = env->GetArrayLength(row);
+ int j = 0;
+ for (j = 0; j < length; j++) {
+ jobject element = env->GetObjectArrayElement(row, j);
+ char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+ printf("%s\t", str);
+ }
+ printf("\n");
+ }
+ carbonReaderClass.close();
+}
+
+// 2.2 read data from S3
+
+/**
+ * read data from S3
+ * parameter is ak sk endpoint
+ *
+ * @param env jni env
+ * @param argv argument vector
+ * @return
+ */
+bool readFromS3(JNIEnv *env, char *argv[]) {
+ CarbonReader reader;
+
+ char *args[3];
+ // "your access key"
+ args[0] = argv[1];
+ // "your secret key"
+ args[1] = argv[2];
+ // "your endPoint"
+ args[2] = argv[3];
+
+ reader.builder(env, "s3a://sdk/WriterOutput", "test");
+ reader.withHadoopConf(3, args);
+ reader.build();
+ printf("\nRead data from S3:\n");
+ while (reader.hasNext()) {
+ jobjectArray row = reader.readNextRow();
+ jsize length = env->GetArrayLength(row);
+
+ int j = 0;
+ for (j = 0; j < length; j++) {
+ jobject element = env->GetObjectArrayElement(row, j);
+ char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+ printf("%s\t", str);
+ }
+ printf("\n");
+ }
+
+ reader.close();
+}
+
+// 3. destory JVM
+ (jvm)->DestroyJavaVM();
+</code></pre>
+<p>Find example code at main.cpp of CSDK module</p>
+<h2>
+<a id="api-list" class="anchor" href="#api-list" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>API List</h2>
+<pre><code> /**
+ * create a CarbonReaderBuilder object for building carbonReader,
+ * CarbonReaderBuilder object can configure different parameter
+ *
+ * @param env JNIEnv
+ * @param path data store path
+ * @param tableName table name
+ * @return CarbonReaderBuilder object
+ */
+ jobject builder(JNIEnv *env, char *path, char *tableName);
+
+ /**
+ * Configure the projection column names of carbon reader
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return CarbonReaderBuilder object
+ */
+ jobject projection(int argc, char *argv[]);
+
+ /**
+ * build carbon reader with argument vector
+ * it support multiple parameter
+ * like: key=value
+ * for example: fs.s3a.access.key=XXXX, XXXX is user's access key value
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return CarbonReaderBuilder object
+ **/
+ jobject withHadoopConf(int argc, char *argv[]);
+
+ /**
+ * build carbonReader object for reading data
+ * it support read data from load disk
+ *
+ * @return carbonReader object
+ */
+ jobject build();
+
+ /**
+ * Whether it has next row data
+ *
+ * @return boolean value, if it has next row, return true. if it hasn't next row, return false.
+ */
+ jboolean hasNext();
+
+ /**
+ * read next row from data
+ *
+ * @return object array of one row
+ */
+ jobjectArray readNextRow();
+
+ /**
+ * close the carbon reader
+ *
+ * @return boolean value
+ */
+ jboolean close();
+
+</code></pre>
+<script>
+$(function() {
+ // Show selected style on nav item
+ $('.b-nav__api').addClass('selected');
+
+ if (!$('.b-nav__api').parent().hasClass('nav__item__with__subs--expanded')) {
+ // Display api subnav items
+ $('.b-nav__api').parent().toggleClass('nav__item__with__subs--expanded');
+ }
+});
+</script></div>
+</div>
+</div>
+</div>
+<div class="doc-footer">
+ <a href="#top" class="scroll-top">Top</a>
+</div>
+</div>
+</section>
+</div>
+</div>
+</div>
+</section><!-- End systemblock part -->
+<script src="js/custom.js"></script>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/WEB-INF/classes/application.conf
----------------------------------------------------------------------
diff --git a/content/WEB-INF/classes/application.conf b/content/WEB-INF/classes/application.conf
index ba425e5..020b127 100644
--- a/content/WEB-INF/classes/application.conf
+++ b/content/WEB-INF/classes/application.conf
@@ -16,7 +16,9 @@ fileList=["configuration-parameters",
"release-guide",
"how-to-contribute-to-apache-carbondata",
"introduction",
- "usecases"
+ "usecases",
+ "CSDK-guide",
+ "carbon-as-spark-datasource-guide"
]
dataMapFileList=[
"bloomfilter-datamap-guide",
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/bloomfilter-datamap-guide.html
----------------------------------------------------------------------
diff --git a/content/bloomfilter-datamap-guide.html b/content/bloomfilter-datamap-guide.html
index 8030599..2ecfa44 100644
--- a/content/bloomfilter-datamap-guide.html
+++ b/content/bloomfilter-datamap-guide.html
@@ -52,6 +52,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
target="_blank">Apache CarbonData 1.4.1</a></li>
<li>
@@ -179,7 +182,12 @@
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
</div>
- <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a>
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
@@ -211,7 +219,7 @@
<div class="col-sm-12 col-md-12">
<div>
<h1>
-<a id="carbondata-bloomfilter-datamap-alpha-feature" class="anchor" href="#carbondata-bloomfilter-datamap-alpha-feature" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData BloomFilter DataMap (Alpha Feature)</h1>
+<a id="carbondata-bloomfilter-datamap" class="anchor" href="#carbondata-bloomfilter-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData BloomFilter DataMap</h1>
<ul>
<li><a href="#datamap-management">DataMap Management</a></li>
<li><a href="#bloomfilter-datamap-introduction">BloomFilter Datamap Introduction</a></li>
@@ -238,7 +246,7 @@ ON TABLE main_table
</code></pre>
<p>Disable Datamap</p>
<blockquote>
-<p>The datamap by default is enabled. To support tuning on query, we can disable a specific datamap during query to observe whether we can gain performance enhancement from it. This will only take effect current session.</p>
+<p>The datamap by default is enabled. To support tuning on query, we can disable a specific datamap during query to observe whether we can gain performance enhancement from it. This is effective only for current session.</p>
</blockquote>
<pre><code>// disable the datamap
SET carbon.datamap.visible.dbName.tableName.dataMapName = false
@@ -268,7 +276,7 @@ and we always query on <code>id</code> and <code>name</code> with precise value.
since <code>id</code> is in the sort_columns and it is orderd,
query on it will be fast because CarbonData can skip all the irrelative blocklets.
But queries on <code>name</code> may be bad since the blocklet minmax may not help,
-because in each blocklet the range of the value of <code>name</code> may be the same -- all from A<em>~z</em>.
+because in each blocklet the range of the value of <code>name</code> may be the same -- all from A* to z*.
In this case, user can create a BloomFilter datamap on column <code>name</code>.
Moreover, user can also create a BloomFilter datamap on the sort_columns.
This is useful if user has too many segments and the range of the value of sort_columns are almost the same.</p>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/carbon-as-spark-datasource-guide.html
----------------------------------------------------------------------
diff --git a/content/carbon-as-spark-datasource-guide.html b/content/carbon-as-spark-datasource-guide.html
new file mode 100644
index 0000000..e29f120
--- /dev/null
+++ b/content/carbon-as-spark-datasource-guide.html
@@ -0,0 +1,349 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'>
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <title>CarbonData</title>
+ <style>
+
+ </style>
+ <!-- Bootstrap -->
+
+ <link rel="stylesheet" href="css/bootstrap.min.css">
+ <link href="css/style.css" rel="stylesheet">
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="js/jquery.min.js"></script>
+ <script src="js/bootstrap.min.js"></script>
+ <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js"></script>
+
+
+</head>
+<body>
+<header>
+ <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper">
+ <div class="container">
+ <div class="navbar-header">
+ <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse"
+ class="navbar-toggle collapsed" type="button">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="index.html" class="logo">
+ <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/>
+ </a>
+ </div>
+ <div class="navbar-collapse collapse cd_navcontnt" id="navbar">
+ <ul class="nav navbar-nav navbar-right navlist-custom">
+ <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a>
+ </li>
+ <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false"> Download <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
+ target="_blank">Apache CarbonData 1.4.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
+ target="_blank">Apache CarbonData 1.4.0</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
+ target="_blank">Apache CarbonData 1.3.1</a></li>
+ <li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
+ target="_blank">Apache CarbonData 1.3.0</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
+ target="_blank">Release Archive</a></li>
+ </ul>
+ </li>
+ <li><a href="documentation.html" class="active">Documentation</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
+ aria-expanded="false">Community <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"
+ target="_blank">Contributing to CarbonData</a></li>
+ <li>
+ <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"
+ target="_blank">Release Guide</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"
+ target="_blank">Project PMC and Committers</a></li>
+ <li>
+ <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"
+ target="_blank">CarbonData Meetups</a></li>
+ <li><a href="security.html">Apache CarbonData Security</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/CARBONDATA" target="_blank">Apache
+ Jira</a></li>
+ <li><a href="videogallery.html">CarbonData Videos </a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="apache_link hidden-xs dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="http://www.apache.org/" class="hidden-lg hidden-md hidden-sm dropdown-toggle"
+ data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html"
+ target="_blank">Sponsorship</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
+ </ul>
+ </li>
+
+ <li>
+ <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a>
+
+ </li>
+
+ </ul>
+ </div><!--/.nav-collapse -->
+ <div id="search-box">
+ <form method="get" action="http://www.google.com/search" target="_blank">
+ <div class="search-block">
+ <table border="0" cellpadding="0" width="100%">
+ <tr>
+ <td style="width:80%">
+ <input type="text" name="q" size=" 5" maxlength="255" value=""
+ class="search-input" placeholder="Search...." required/>
+ </td>
+ <td style="width:20%">
+ <input type="submit" value="Search"/></td>
+ </tr>
+ <tr>
+ <td align="left" style="font-size:75%" colspan="2">
+ <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/>
+ <span style=" position: relative; top: -3px;"> Only search for CarbonData</span>
+ </td>
+ </tr>
+ </table>
+ </div>
+ </form>
+ </div>
+ </div>
+ </nav>
+</header> <!-- end Header part -->
+
+<div class="fixed-padding"></div> <!-- top padding with fixde header -->
+
+<section><!-- Dashboard nav -->
+ <div class="container-fluid q">
+ <div class="col-sm-12 col-md-12 maindashboard">
+ <div class="verticalnavbar">
+ <nav class="b-sticky-nav">
+ <div class="nav-scroller">
+ <div class="nav__inner">
+ <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a>
+ <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a>
+ <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a>
+ <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a>
+ <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a>
+ <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a>
+ <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a>
+ <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a>
+ <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a>
+ </div>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a>
+ <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a>
+ <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a>
+ <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a>
+ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
+ </div>
+
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
+ <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
+ <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
+ <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
+ <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
+ <a class="b-nav__security nav__item" href="./security.html">Security</a>
+ <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a>
+ </div>
+ </div>
+ <div class="navindicator">
+ <div class="b-nav__intro navindicator__item"></div>
+ <div class="b-nav__quickstart navindicator__item"></div>
+ <div class="b-nav__uses navindicator__item"></div>
+ <div class="b-nav__docs navindicator__item"></div>
+ <div class="b-nav__datamap navindicator__item"></div>
+ <div class="b-nav__api navindicator__item"></div>
+ <div class="b-nav__perf navindicator__item"></div>
+ <div class="b-nav__s3 navindicator__item"></div>
+ <div class="b-nav__faq navindicator__item"></div>
+ <div class="b-nav__contri navindicator__item"></div>
+ <div class="b-nav__security navindicator__item"></div>
+ </div>
+ </nav>
+ </div>
+ <div class="mdcontent">
+ <section>
+ <div style="padding:10px 15px;">
+ <div id="viewpage" name="viewpage">
+ <div class="row">
+ <div class="col-sm-12 col-md-12">
+ <div>
+<h1>
+<a id="carbondata-as-sparks-datasource" class="anchor" href="#carbondata-as-sparks-datasource" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData as Spark's Datasource</h1>
+<p>The CarbonData fileformat is now integrated as Spark datasource for read and write operation without using CarbonSession. This is useful for users who wants to use carbondata as spark's data source.</p>
+<p><strong>Note:</strong> You can only apply the functions/features supported by spark datasource APIs, functionalities supported would be similar to Parquet. The carbon session features are not supported.</p>
+<h1>
+<a id="create-table-with-ddl" class="anchor" href="#create-table-with-ddl" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create Table with DDL</h1>
+<p>Now you can create Carbon table using Spark's datasource DDL syntax.</p>
+<pre><code> CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db_name.]table_name
+ [(col_name1 col_type1 [COMMENT col_comment1], ...)]
+ USING CARBON
+ [OPTIONS (key1=val1, key2=val2, ...)]
+ [PARTITIONED BY (col_name1, col_name2, ...)]
+ [CLUSTERED BY (col_name3, col_name4, ...) INTO num_buckets BUCKETS]
+ [LOCATION path]
+ [COMMENT table_comment]
+ [TBLPROPERTIES (key1=val1, key2=val2, ...)]
+ [AS select_statement]
+</code></pre>
+<h2>
+<a id="supported-options" class="anchor" href="#supported-options" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Supported OPTIONS</h2>
+<table>
+<thead>
+<tr>
+<th>Property</th>
+<th>Default Value</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>table_blocksize</td>
+<td>1024</td>
+<td>Size of blocks to write onto hdfs. For more details, see <a href="./ddl-of-carbondata.html#table-block-size-configuration">Table Block Size Configuration</a>.</td>
+</tr>
+<tr>
+<td>table_blocklet_size</td>
+<td>64</td>
+<td>Size of blocklet to write.</td>
+</tr>
+<tr>
+<td>local_dictionary_threshold</td>
+<td>10000</td>
+<td>Cardinality upto which the local dictionary can be generated. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
+</tr>
+<tr>
+<td>local_dictionary_enable</td>
+<td>false</td>
+<td>Enable local dictionary generation. For more details, see <a href="./ddl-of-carbondata.html#local-dictionary-configuration">Local Dictionary Configuration</a>.</td>
+</tr>
+<tr>
+<td>sort_columns</td>
+<td>all dimensions are sorted</td>
+<td>Columns to include in sort and its order of sort. For more details, see <a href="./ddl-of-carbondata.html#sort-columns-configuration">Sort Columns Configuration</a>.</td>
+</tr>
+<tr>
+<td>sort_scope</td>
+<td>local_sort</td>
+<td>Sort scope of the load.Options include no sort, local sort, batch sort, and global sort. For more details, see <a href="./ddl-of-carbondata.html#sort-scope-configuration">Sort Scope Configuration</a>.</td>
+</tr>
+<tr>
+<td>long_string_columns</td>
+<td>null</td>
+<td>Comma separated string/char/varchar columns which are more than 32k length. For more details, see <a href="./ddl-of-carbondata.html#string-longer-than-32000-characters">String longer than 32000 characters</a>.</td>
+</tr>
+</tbody>
+</table>
+<h2>
+<a id="example" class="anchor" href="#example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example</h2>
+<pre><code> CREATE TABLE CARBON_TABLE (NAME STRING) USING CARBON OPTIONS('table_block_size'='256')
+</code></pre>
+<h1>
+<a id="using-dataframe" class="anchor" href="#using-dataframe" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Using DataFrame</h1>
+<p>Carbon format can be used in dataframe also. Following are the ways to use carbon format in dataframe.</p>
+<p>Write carbon using dataframe</p>
+<pre><code>df.write.format("carbon").save(path)
+</code></pre>
+<p>Read carbon using dataframe</p>
+<pre><code>val df = spark.read.format("carbon").load(path)
+</code></pre>
+<h2>
+<a id="example-1" class="anchor" href="#example-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example</h2>
+<pre><code>import org.apache.spark.sql.SparkSession
+
+val spark = SparkSession
+ .builder()
+ .appName("Spark SQL basic example")
+ .config("spark.some.config.option", "some-value")
+ .getOrCreate()
+
+// For implicit conversions like converting RDDs to DataFrames
+import spark.implicits._
+val df = spark.sparkContext.parallelize(1 to 10 * 10 * 1000)
+ .map(x => (r.nextInt(100000), "name" + x % 8, "city" + x % 50, BigDecimal.apply(x % 60)))
+ .toDF("ID", "name", "city", "age")
+
+// Write to carbon format
+df.write.format("carbon").save("/user/person_table")
+
+// Read carbon using dataframe
+val dfread = spark.read.format("carbon").load("/user/person_table")
+dfread.show()
+</code></pre>
+<p>Reference : <a href="./configuration-parameters.html">list of carbon properties</a></p>
+<script>
+$(function() {
+ // Show selected style on nav item
+ $('.b-nav__docs').addClass('selected');
+
+ // Display docs subnav items
+ if (!$('.b-nav__docs').parent().hasClass('nav__item__with__subs--expanded')) {
+ $('.b-nav__docs').parent().toggleClass('nav__item__with__subs--expanded');
+ }
+});
+</script></div>
+</div>
+</div>
+</div>
+<div class="doc-footer">
+ <a href="#top" class="scroll-top">Top</a>
+</div>
+</div>
+</section>
+</div>
+</div>
+</div>
+</section><!-- End systemblock part -->
+<script src="js/custom.js"></script>
+</body>
+</html>
\ No newline at end of file