You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2018/10/17 10:14:57 UTC
[09/20] carbondata-site git commit: modified for 1.5.0 version
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/sdk-guide.html
----------------------------------------------------------------------
diff --git a/content/sdk-guide.html b/content/sdk-guide.html
index a252965..5ddf9f7 100644
--- a/content/sdk-guide.html
+++ b/content/sdk-guide.html
@@ -52,6 +52,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
target="_blank">Apache CarbonData 1.4.1</a></li>
<li>
@@ -179,7 +182,12 @@
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
</div>
- <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a>
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
@@ -259,9 +267,9 @@ These SDK writer output contains just a carbondata and carbonindex files. No met
<span class="pl-smi">CarbonProperties</span><span class="pl-k">.</span>getInstance()<span class="pl-k">.</span>addProperty(<span class="pl-s"><span class="pl-pds">"</span>enable.offheap.sort<span class="pl-pds">"</span></span>, enableOffheap);
- <span class="pl-smi">CarbonWriterBuilder</span> builder <span class="pl-k">=</span> <span class="pl-smi">CarbonWriter</span><span class="pl-k">.</span>builder()<span class="pl-k">.</span>outputPath(path);
+ <span class="pl-smi">CarbonWriterBuilder</span> builder <span class="pl-k">=</span> <span class="pl-smi">CarbonWriter</span><span class="pl-k">.</span>builder()<span class="pl-k">.</span>outputPath(path)<span class="pl-k">.</span>withCsvInput(schema);
- <span class="pl-smi">CarbonWriter</span> writer <span class="pl-k">=</span> builder<span class="pl-k">.</span>buildWriterForCSVInput(schema);
+ <span class="pl-smi">CarbonWriter</span> writer <span class="pl-k">=</span> builder<span class="pl-k">.</span>build();
<span class="pl-k">int</span> rows <span class="pl-k">=</span> <span class="pl-c1">5</span>;
<span class="pl-k">for</span> (<span class="pl-k">int</span> i <span class="pl-k">=</span> <span class="pl-c1">0</span>; i <span class="pl-k"><</span> rows; i<span class="pl-k">++</span>) {
@@ -314,7 +322,7 @@ These SDK writer output contains just a carbondata and carbonindex files. No met
<span class="pl-k">try</span> {
<span class="pl-smi">CarbonWriter</span> writer <span class="pl-k">=</span> <span class="pl-smi">CarbonWriter</span><span class="pl-k">.</span>builder()
.outputPath(path)
- .buildWriterForAvroInput(<span class="pl-k">new</span> <span class="pl-smi">org.apache.avro<span class="pl-k">.</span>Schema</span>.<span class="pl-smi">Parser</span>()<span class="pl-k">.</span>parse(avroSchema));
+ .withAvroInput(<span class="pl-k">new</span> <span class="pl-smi">org.apache.avro<span class="pl-k">.</span>Schema</span>.<span class="pl-smi">Parser</span>()<span class="pl-k">.</span>parse(avroSchema))<span class="pl-k">.</span>build();
<span class="pl-k">for</span> (<span class="pl-k">int</span> i <span class="pl-k">=</span> <span class="pl-c1">0</span>; i <span class="pl-k"><</span> <span class="pl-c1">100</span>; i<span class="pl-k">++</span>) {
writer<span class="pl-k">.</span>write(record);
@@ -352,10 +360,10 @@ These SDK writer output contains just a carbondata and carbonindex files. No met
<span class="pl-smi">Schema</span> <span class="pl-smi">CarbonSchema</span> <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Schema</span>(fields);
- <span class="pl-smi">CarbonWriterBuilder</span> builder <span class="pl-k">=</span> <span class="pl-smi">CarbonWriter</span><span class="pl-k">.</span>builder()<span class="pl-k">.</span>outputPath(path);
+ <span class="pl-smi">CarbonWriterBuilder</span> builder <span class="pl-k">=</span> <span class="pl-smi">CarbonWriter</span><span class="pl-k">.</span>builder()<span class="pl-k">.</span>outputPath(path)<span class="pl-k">.</span>withJsonInput(<span class="pl-smi">CarbonSchema</span>);
<span class="pl-c"><span class="pl-c">//</span> initialize json writer with carbon schema</span>
- <span class="pl-smi">CarbonWriter</span> writer <span class="pl-k">=</span> builder<span class="pl-k">.</span>buildWriterForJsonInput(<span class="pl-smi">CarbonSchema</span>);
+ <span class="pl-smi">CarbonWriter</span> writer <span class="pl-k">=</span> builder<span class="pl-k">.</span>build();
<span class="pl-c"><span class="pl-c">//</span> one row of json Data as String</span>
<span class="pl-smi">String</span> <span class="pl-smi">JsonRow</span> <span class="pl-k">=</span> <span class="pl-s"><span class="pl-pds">"</span>{<span class="pl-cce">\"</span>name<span class="pl-cce">\"</span>:<span class="pl-cce">\"</span>abcd<span class="pl-cce">\"</span>, <span class="pl-cce">\"</span>age<span class="pl-cce">\"</span>:10}<span class="pl-pds">"</span></span>;
@@ -368,59 +376,127 @@ These SDK writer output contains just a carbondata and carbonindex files. No met
} </pre></div>
<h2>
<a id="datatypes-mapping" class="anchor" href="#datatypes-mapping" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Datatypes Mapping</h2>
-<p>Each of SQL data types are mapped into data types of SDK. Following are the mapping:</p>
+<p>Each of SQL data types and Avro Data Types are mapped into data types of SDK. Following are the mapping:</p>
<table>
<thead>
<tr>
<th>SQL DataTypes</th>
+<th>Avro DataTypes</th>
<th>Mapped SDK DataTypes</th>
</tr>
</thead>
<tbody>
<tr>
<td>BOOLEAN</td>
+<td>BOOLEAN</td>
<td>DataTypes.BOOLEAN</td>
</tr>
<tr>
<td>SMALLINT</td>
+<td>-</td>
<td>DataTypes.SHORT</td>
</tr>
<tr>
<td>INTEGER</td>
+<td>INTEGER</td>
<td>DataTypes.INT</td>
</tr>
<tr>
<td>BIGINT</td>
+<td>LONG</td>
<td>DataTypes.LONG</td>
</tr>
<tr>
<td>DOUBLE</td>
+<td>DOUBLE</td>
<td>DataTypes.DOUBLE</td>
</tr>
<tr>
<td>VARCHAR</td>
+<td>-</td>
<td>DataTypes.STRING</td>
</tr>
<tr>
+<td>FLOAT</td>
+<td>FLOAT</td>
+<td>DataTypes.FLOAT</td>
+</tr>
+<tr>
+<td>BYTE</td>
+<td>-</td>
+<td>DataTypes.BYTE</td>
+</tr>
+<tr>
+<td>DATE</td>
<td>DATE</td>
<td>DataTypes.DATE</td>
</tr>
<tr>
<td>TIMESTAMP</td>
+<td>-</td>
<td>DataTypes.TIMESTAMP</td>
</tr>
<tr>
<td>STRING</td>
+<td>STRING</td>
<td>DataTypes.STRING</td>
</tr>
<tr>
<td>DECIMAL</td>
+<td>DECIMAL</td>
<td>DataTypes.createDecimalType(precision, scale)</td>
</tr>
+<tr>
+<td>ARRAY</td>
+<td>ARRAY</td>
+<td>DataTypes.createArrayType(elementType)</td>
+</tr>
+<tr>
+<td>STRUCT</td>
+<td>RECORD</td>
+<td>DataTypes.createStructType(fields)</td>
+</tr>
+<tr>
+<td>-</td>
+<td>ENUM</td>
+<td>DataTypes.STRING</td>
+</tr>
+<tr>
+<td>-</td>
+<td>UNION</td>
+<td>DataTypes.createStructType(types)</td>
+</tr>
+<tr>
+<td>-</td>
+<td>MAP</td>
+<td>DataTypes.createMapType(keyType, valueType)</td>
+</tr>
+<tr>
+<td>-</td>
+<td>TimeMillis</td>
+<td>DataTypes.INT</td>
+</tr>
+<tr>
+<td>-</td>
+<td>TimeMicros</td>
+<td>DataTypes.LONG</td>
+</tr>
+<tr>
+<td>-</td>
+<td>TimestampMillis</td>
+<td>DataTypes.TIMESTAMP</td>
+</tr>
+<tr>
+<td>-</td>
+<td>TimestampMicros</td>
+<td>DataTypes.TIMESTAMP</td>
+</tr>
</tbody>
</table>
-<p><strong>NOTE:</strong>
-Carbon Supports below logical types of AVRO.
+<p><strong>NOTE:</strong></p>
+<ol>
+<li>
+<p>Carbon Supports below logical types of AVRO.
a. Date
The date logical type represents a date within the calendar, with no reference to a particular time zone or time of day.
A date logical type annotates an Avro int, where the int stores the number of days from the unix epoch, 1 January 1970 (ISO calendar).
@@ -429,10 +505,27 @@ The timestamp-millis logical type represents an instant on the global timeline,
A timestamp-millis logical type annotates an Avro long, where the long stores the number of milliseconds from the unix epoch, 1 January 1970 00:00:00.000 UTC.
c. Timestamp (microsecond precision)
The timestamp-micros logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one microsecond.
-A timestamp-micros logical type annotates an Avro long, where the long stores the number of microseconds from the unix epoch, 1 January 1970 00:00:00.000000 UTC.</p>
-<pre><code>Currently the values of logical types are not validated by carbon.
-Expect that avro record passed by the user is already validated by avro record generator tools.
-</code></pre>
+A timestamp-micros logical type annotates an Avro long, where the long stores the number of microseconds from the unix epoch, 1 January 1970 00:00:00.000000 UTC.
+d. Decimal
+The decimal logical type represents an arbitrary-precision signed decimal number of the form unscaled � 10-scale.
+A decimal logical type annotates Avro bytes or fixed types. The byte array must contain the two's-complement representation of the unscaled integer value in big-endian byte order. The scale is fixed, and is specified using an attribute.
+e. Time (millisecond precision)
+The time-millis logical type represents a time of day, with no reference to a particular calendar, time zone or date, with a precision of one millisecond.
+A time-millis logical type annotates an Avro int, where the int stores the number of milliseconds after midnight, 00:00:00.000.
+f. Time (microsecond precision)
+The time-micros logical type represents a time of day, with no reference to a particular calendar, time zone or date, with a precision of one microsecond.
+A time-micros logical type annotates an Avro long, where the long stores the number of microseconds after midnight, 00:00:00.000000.</p>
+<p>Currently the values of logical types are not validated by carbon.
+Expect that avro record passed by the user is already validated by avro record generator tools.</p>
+</li>
+<li>
+<p>If the string data is more than 32K in length, use withTableProperties() with "long_string_columns" property
+or directly use DataTypes.VARCHAR if it is carbon schema.</p>
+</li>
+<li>
+<p>Avro Bytes, Fixed and Duration data types are not yet supported.</p>
+</li>
+</ol>
<h2>
<a id="run-sql-on-files-directly" class="anchor" href="#run-sql-on-files-directly" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Run SQL on files directly</h2>
<p>Instead of creating table and query it, you can also query that file directly with SQL.</p>
@@ -454,18 +547,6 @@ Expect that avro record passed by the user is already validated by avro record g
public CarbonWriterBuilder outputPath(String path);
</code></pre>
<pre><code>/**
-* If set false, writes the carbondata and carbonindex files in a flat folder structure
-* @param isTransactionalTable is a boolelan value
-* if set to false, then writes the carbondata and carbonindex files
-* in a flat folder structure.
-* if set to true, then writes the carbondata and carbonindex files
-* in segment folder structure..
-* By default set to false.
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder isTransactionalTable(boolean isTransactionalTable);
-</code></pre>
-<pre><code>/**
* to set the timestamp in the carbondata and carbonindex index files
* @param UUID is a timestamp to be used in the carbondata and carbonindex index files.
* By default set to zero.
@@ -511,14 +592,6 @@ public CarbonWriterBuilder localDictionaryThreshold(int localDictionaryThreshold
public CarbonWriterBuilder sortBy(String[] sortColumns);
</code></pre>
<pre><code>/**
-* If set, create a schema file in metadata folder.
-* @param persist is a boolean value, If set to true, creates a schema file in metadata folder.
-* By default set to false. will not create metadata folder
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder persistSchemaFile(boolean persist);
-</code></pre>
-<pre><code>/**
* sets the taskNo for the writer. SDKs concurrently running
* will set taskNo in order to avoid conflicts in file's name during write.
* @param taskNo is the TaskNo user wants to specify.
@@ -540,7 +613,7 @@ public CarbonWriterBuilder taskNo(long taskNo);
* g. complex_delimiter_level_2 -- value to Split the nested complexTypeData
* h. quotechar
* i. escapechar
-*
+*
* Default values are as follows.
*
* a. bad_records_logger_enable -- "false"
@@ -558,84 +631,76 @@ public CarbonWriterBuilder taskNo(long taskNo);
public CarbonWriterBuilder withLoadOptions(Map<String, String> options);
</code></pre>
<pre><code>/**
- * To support the table properties for sdk writer
- *
- * @param options key,value pair of create table properties.
- * supported keys values are
- * a. blocksize -- [1-2048] values in MB. Default value is 1024
- * b. blockletsize -- values in MB. Default value is 64 MB
- * c. localDictionaryThreshold -- positive value, default is 10000
- * d. enableLocalDictionary -- true / false. Default is false
- * e. sortcolumns -- comma separated column. "c1,c2". Default all dimensions are sorted.
- *
- * @return updated CarbonWriterBuilder
- */
+* To support the table properties for sdk writer
+*
+* @param options key,value pair of create table properties.
+* supported keys values are
+* a. table_blocksize -- [1-2048] values in MB. Default value is 1024
+* b. table_blocklet_size -- values in MB. Default value is 64 MB
+* c. local_dictionary_threshold -- positive value, default is 10000
+* d. local_dictionary_enable -- true / false. Default is false
+* e. sort_columns -- comma separated column. "c1,c2". Default all dimensions are sorted.
+ If empty string "" is passed. No columns are sorted
+* j. sort_scope -- "local_sort", "no_sort", "batch_sort". default value is "local_sort"
+* k. long_string_columns -- comma separated string columns which are more than 32k length.
+* default value is null.
+*
+* @return updated CarbonWriterBuilder
+*/
public CarbonWriterBuilder withTableProperties(Map<String, String> options);
</code></pre>
<pre><code>/**
-* this writer is not thread safe, use buildThreadSafeWriterForCSVInput in multi thread environment
-* Build a {@link CarbonWriter}, which accepts row in CSV format object
-* @param schema carbon Schema object {org.apache.carbondata.sdk.file.Schema}
-* @return CSVCarbonWriter
-* @throws IOException
-* @throws InvalidLoadOptionException
+* To make sdk writer thread safe.
+*
+* @param numOfThreads should number of threads in which writer is called in multi-thread scenario
+* default sdk writer is not thread safe.
+* can use one writer instance in one thread only.
+* @return updated CarbonWriterBuilder
*/
-public CarbonWriter buildWriterForCSVInput(org.apache.carbondata.sdk.file.Schema schema) throws IOException, InvalidLoadOptionException;
+public CarbonWriterBuilder withThreadSafe(short numOfThreads);
</code></pre>
<pre><code>/**
-* Can use this writer in multi-thread instance.
-* Build a {@link CarbonWriter}, which accepts row in CSV format
-* @param schema carbon Schema object {org.apache.carbondata.sdk.file.Schema}
-* @param numOfThreads number of threads() in which .write will be called.
-* @return CSVCarbonWriter
-* @throws IOException
-* @throws InvalidLoadOptionException
+* To support hadoop configuration
+*
+* @param conf hadoop configuration support, can set s3a AK,SK,end point and other conf with this
+* @return updated CarbonWriterBuilder
*/
-public CarbonWriter buildThreadSafeWriterForCSVInput(Schema schema, short numOfThreads)
- throws IOException, InvalidLoadOptionException;
+public CarbonWriterBuilder withHadoopConf(Configuration conf)
</code></pre>
<pre><code>/**
-* this writer is not thread safe, use buildThreadSafeWriterForAvroInput in multi thread environment
-* Build a {@link CarbonWriter}, which accepts Avro format object
-* @param avroSchema avro Schema object {org.apache.avro.Schema}
-* @return AvroCarbonWriter
-* @throws IOException
-* @throws InvalidLoadOptionException
+* to build a {@link CarbonWriter}, which accepts row in CSV format
+*
+* @param schema carbon Schema object {org.apache.carbondata.sdk.file.Schema}
+* @return CarbonWriterBuilder
*/
-public CarbonWriter buildWriterForAvroInput(org.apache.avro.Schema schema) throws IOException, InvalidLoadOptionException;
+public CarbonWriterBuilder withCsvInput(Schema schema);
</code></pre>
<pre><code>/**
-* Can use this writer in multi-thread instance.
-* Build a {@link CarbonWriter}, which accepts Avro object
+* to build a {@link CarbonWriter}, which accepts Avro object
+*
* @param avroSchema avro Schema object {org.apache.avro.Schema}
-* @param numOfThreads number of threads() in which .write will be called.
-* @return AvroCarbonWriter
-* @throws IOException
-* @throws InvalidLoadOptionException
+* @return CarbonWriterBuilder
*/
-public CarbonWriter buildThreadSafeWriterForAvroInput(org.apache.avro.Schema avroSchema, short numOfThreads)
- throws IOException, InvalidLoadOptionException
+public CarbonWriterBuilder withAvroInput(org.apache.avro.Schema avroSchema);
</code></pre>
<pre><code>/**
-* this writer is not thread safe, use buildThreadSafeWriterForJsonInput in multi thread environment
-* Build a {@link CarbonWriter}, which accepts Json object
+* to build a {@link CarbonWriter}, which accepts Json object
+*
* @param carbonSchema carbon Schema object
-* @return JsonCarbonWriter
-* @throws IOException
-* @throws InvalidLoadOptionException
+* @return CarbonWriterBuilder
*/
-public JsonCarbonWriter buildWriterForJsonInput(Schema carbonSchema);
+public CarbonWriterBuilder withJsonInput(Schema carbonSchema);
</code></pre>
<pre><code>/**
-* Can use this writer in multi-thread instance.
-* Build a {@link CarbonWriter}, which accepts Json object
-* @param carbonSchema carbon Schema object
-* @param numOfThreads number of threads() in which .write will be called.
-* @return JsonCarbonWriter
+* Build a {@link CarbonWriter}
+* This writer is not thread safe,
+* use withThreadSafe() configuration in multi thread environment
+*
+* @return CarbonWriter {AvroCarbonWriter/CSVCarbonWriter/JsonCarbonWriter based on Input Type }
* @throws IOException
* @throws InvalidLoadOptionException
*/
-public JsonCarbonWriter buildThreadSafeWriterForJsonInput(Schema carbonSchema, short numOfThreads)
+public CarbonWriter build() throws IOException, InvalidLoadOptionException;
</code></pre>
<h3>
<a id="class-orgapachecarbondatasdkfilecarbonwriter" class="anchor" href="#class-orgapachecarbondatasdkfilecarbonwriter" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.CarbonWriter</h3>
@@ -645,7 +710,6 @@ public JsonCarbonWriter buildThreadSafeWriterForJsonInput(Schema carbonSchema, s
* which is one row of data.
* If CSVCarbonWriter, object is of type String[], which is one row of data
* If JsonCarbonWriter, object is of type String, which is one row of json
-* Note: This API is not thread safe if writer is not built with number of threads argument.
* @param object
* @throws IOException
*/
@@ -793,17 +857,6 @@ External client can make use of this reader to read CarbonData files without Car
*/
public CarbonReaderBuilder projection(String[] projectionColumnNames);
</code></pre>
-<pre><code> /**
- * Configure the transactional status of table
- * If set to false, then reads the carbondata and carbonindex files from a flat folder structure.
- * If set to true, then reads the carbondata and carbonindex files from segment folder structure.
- * Default value is false
- *
- * @param isTransactionalTable whether is transactional table or not
- * @return CarbonReaderBuilder object
- */
- public CarbonReaderBuilder isTransactionalTable(boolean isTransactionalTable);
-</code></pre>
<pre><code> /**
* Configure the filter expression for carbon reader
*
@@ -812,56 +865,13 @@ External client can make use of this reader to read CarbonData files without Car
*/
public CarbonReaderBuilder filter(Expression filterExpression);
</code></pre>
-<pre><code> /**
- * Set the access key for S3
- *
- * @param key the string of access key for different S3 type,like: fs.s3a.access.key
- * @param value the value of access key
- * @return CarbonWriterBuilder
- */
- public CarbonReaderBuilder setAccessKey(String key, String value);
-</code></pre>
-<pre><code> /**
- * Set the access key for S3.
- *
- * @param value the value of access key
- * @return CarbonWriterBuilder object
- */
- public CarbonReaderBuilder setAccessKey(String value);
-</code></pre>
-<pre><code> /**
- * Set the secret key for S3
- *
- * @param key the string of secret key for different S3 type,like: fs.s3a.secret.key
- * @param value the value of secret key
- * @return CarbonWriterBuilder object
- */
- public CarbonReaderBuilder setSecretKey(String key, String value);
-</code></pre>
-<pre><code> /**
- * Set the secret key for S3
- *
- * @param value the value of secret key
- * @return CarbonWriterBuilder object
- */
- public CarbonReaderBuilder setSecretKey(String value);
-</code></pre>
-<pre><code> /**
- * Set the endpoint for S3
- *
- * @param key the string of endpoint for different S3 type,like: fs.s3a.endpoint
- * @param value the value of endpoint
- * @return CarbonWriterBuilder object
- */
- public CarbonReaderBuilder setEndPoint(String key, String value);
-</code></pre>
-<pre><code> /**
- * Set the endpoint for S3
- *
- * @param value the value of endpoint
- * @return CarbonWriterBuilder object
- */
- public CarbonReaderBuilder setEndPoint(String value);
+<pre><code>/**
+ * To support hadoop configuration
+ *
+ * @param conf hadoop configuration support, can set s3a AK,SK,end point and other conf with this
+ * @return updated CarbonReaderBuilder
+ */
+ public CarbonReaderBuilder withHadoopConf(Configuration conf);
</code></pre>
<pre><code> /**
* Build CarbonReader
@@ -992,8 +1002,15 @@ public String getProperty(String key, String defaultValue);
</code></pre>
<p>Reference : <a href="./configuration-parameters.html">list of carbon properties</a></p>
<script>
-// Show selected style on nav item
-$(function() { $('.b-nav__api').addClass('selected'); });
+$(function() {
+ // Show selected style on nav item
+ $('.b-nav__api').addClass('selected');
+
+ if (!$('.b-nav__api').parent().hasClass('nav__item__with__subs--expanded')) {
+ // Display api subnav items
+ $('.b-nav__api').parent().toggleClass('nav__item__with__subs--expanded');
+ }
+});
</script></div>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/sdk-writer-guide.html
----------------------------------------------------------------------
diff --git a/content/sdk-writer-guide.html b/content/sdk-writer-guide.html
deleted file mode 100644
index 36bb9ad..0000000
--- a/content/sdk-writer-guide.html
+++ /dev/null
@@ -1,549 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1">
- <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'>
- <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
- <title>CarbonData</title>
- <style>
-
- </style>
- <!-- Bootstrap -->
-
- <link rel="stylesheet" href="css/bootstrap.min.css">
- <link href="css/style.css" rel="stylesheet">
- <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
- <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
- <!--[if lt IE 9]>
- <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
- <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js"></script>
- <![endif]-->
- <script src="js/jquery.min.js"></script>
- <script src="js/bootstrap.min.js"></script>
-
-
-</head>
-<body>
-<header>
- <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper">
- <div class="container">
- <div class="navbar-header">
- <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse"
- class="navbar-toggle collapsed" type="button">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
- <a href="index.html" class="logo">
- <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/>
- </a>
- </div>
- <div class="navbar-collapse collapse cd_navcontnt" id="navbar">
- <ul class="nav navbar-nav navbar-right navlist-custom">
- <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a>
- </li>
- <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true"
- aria-expanded="false"> Download <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"
- target="_blank">Apache CarbonData 1.4.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"
- target="_blank">Apache CarbonData 1.3.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"
- target="_blank">Apache CarbonData 1.3.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"
- target="_blank">Apache CarbonData 1.2.0</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"
- target="_blank">Apache CarbonData 1.1.1</a></li>
- <li>
- <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"
- target="_blank">Apache CarbonData 1.1.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"
- target="_blank">Apache CarbonData 1.0.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"
- target="_blank">Apache CarbonData 0.2.0</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"
- target="_blank">Apache CarbonData 0.1.1</a></li>
- <li>
- <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"
- target="_blank">Apache CarbonData 0.1.0</a></li>
- <li>
- <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
- target="_blank">Release Archive</a></li>
- </ul>
- </li>
- <li><a href="mainpage.html" class="active">Documentation</a></li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
- aria-expanded="false">Community <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li>
- <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"
- target="_blank">Contributing to CarbonData</a></li>
- <li>
- <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"
- target="_blank">Release Guide</a></li>
- <li>
- <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"
- target="_blank">Project PMC and Committers</a></li>
- <li>
- <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"
- target="_blank">CarbonData Meetups</a></li>
- <li><a href="security.html">Apache CarbonData Security</a></li>
- <li><a href="https://issues.apache.org/jira/browse/CARBONDATA" target="_blank">Apache
- Jira</a></li>
- <li><a href="videogallery.html">CarbonData Videos </a></li>
- </ul>
- </li>
- <li class="dropdown">
- <a href="http://www.apache.org/" class="apache_link hidden-xs dropdown-toggle"
- data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
- <ul class="dropdown-menu">
- <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
- <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
- <li><a href="http://www.apache.org/foundation/sponsorship.html"
- target="_blank">Sponsorship</a></li>
- <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
- </ul>
- </li>
-
- <li class="dropdown">
- <a href="http://www.apache.org/" class="hidden-lg hidden-md hidden-sm dropdown-toggle"
- data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
- <ul class="dropdown-menu">
- <li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
- <li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
- <li><a href="http://www.apache.org/foundation/sponsorship.html"
- target="_blank">Sponsorship</a></li>
- <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
- </ul>
- </li>
-
- <li>
- <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a>
-
- </li>
-
- </ul>
- </div><!--/.nav-collapse -->
- <div id="search-box">
- <form method="get" action="http://www.google.com/search" target="_blank">
- <div class="search-block">
- <table border="0" cellpadding="0" width="100%">
- <tr>
- <td style="width:80%">
- <input type="text" name="q" size=" 5" maxlength="255" value=""
- class="search-input" placeholder="Search...." required/>
- </td>
- <td style="width:20%">
- <input type="submit" value="Search"/></td>
- </tr>
- <tr>
- <td align="left" style="font-size:75%" colspan="2">
- <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/>
- <span style=" position: relative; top: -3px;"> Only search for CarbonData</span>
- </td>
- </tr>
- </table>
- </div>
- </form>
- </div>
- </div>
- </nav>
-</header> <!-- end Header part -->
-
-<div class="fixed-padding"></div> <!-- top padding with fixde header -->
-
-<section><!-- Dashboard nav -->
- <div class="container-fluid q">
- <div class="col-sm-12 col-md-12 maindashboard">
- <div class="row">
- <section>
- <div style="padding:10px 15px;">
- <div id="viewpage" name="viewpage">
- <div class="row">
- <div class="col-sm-12 col-md-12">
- <div><h1>
-<a id="sdk-writer-guide" class="anchor" href="#sdk-writer-guide" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>SDK Writer Guide</h1>
-<p>In the carbon jars package, there exist a carbondata-store-sdk-x.x.x-SNAPSHOT.jar.
-This SDK writer, writes carbondata file and carbonindex file at a given path.
-External client can make use of this writer to convert other format data or live data to create carbondata and index files.
-These SDK writer output contains just a carbondata and carbonindex files. No metadata folder will be present.</p>
-<h2>
-<a id="quick-example" class="anchor" href="#quick-example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Quick example</h2>
-<h3>
-<a id="example-with-csv-format" class="anchor" href="#example-with-csv-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example with csv format</h3>
-<div class="highlight highlight-source-java"><pre> <span class="pl-k">import</span> <span class="pl-smi">java.io.IOException</span>;
-
- <span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException</span>;
- <span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.core.metadata.datatype.DataTypes</span>;
- <span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.CarbonWriter</span>;
- <span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.CarbonWriterBuilder</span>;
- <span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.Field</span>;
- <span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.Schema</span>;
-
- <span class="pl-k">public</span> <span class="pl-k">class</span> <span class="pl-en">TestSdk</span> {
-
- <span class="pl-k">public</span> <span class="pl-k">static</span> <span class="pl-k">void</span> <span class="pl-en">main</span>(<span class="pl-k">String</span>[] <span class="pl-v">args</span>) <span class="pl-k">throws</span> <span class="pl-smi">IOException</span>, <span class="pl-smi">InvalidLoadOptionException</span> {
- testSdkWriter();
- }
-
- <span class="pl-k">public</span> <span class="pl-k">static</span> <span class="pl-k">void</span> <span class="pl-en">testSdkWriter</span>() <span class="pl-k">throws</span> <span class="pl-smi">IOException</span>, <span class="pl-smi">InvalidLoadOptionException</span> {
- <span class="pl-smi">String</span> path <span class="pl-k">=</span> <span class="pl-s"><span class="pl-pds">"</span>/home/root1/Documents/ab/temp<span class="pl-pds">"</span></span>;
-
- <span class="pl-k">Field</span>[] fields <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Field</span>[<span class="pl-c1">2</span>];
- fields[<span class="pl-c1">0</span>] <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Field</span>(<span class="pl-s"><span class="pl-pds">"</span>name<span class="pl-pds">"</span></span>, <span class="pl-smi">DataTypes</span><span class="pl-c1"><span class="pl-k">.</span>STRING</span>);
- fields[<span class="pl-c1">1</span>] <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Field</span>(<span class="pl-s"><span class="pl-pds">"</span>age<span class="pl-pds">"</span></span>, <span class="pl-smi">DataTypes</span><span class="pl-c1"><span class="pl-k">.</span>INT</span>);
-
- <span class="pl-smi">Schema</span> schema <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Schema</span>(fields);
-
- <span class="pl-smi">CarbonWriterBuilder</span> builder <span class="pl-k">=</span> <span class="pl-smi">CarbonWriter</span><span class="pl-k">.</span>builder()<span class="pl-k">.</span>withSchema(schema)<span class="pl-k">.</span>outputPath(path);
-
- <span class="pl-smi">CarbonWriter</span> writer <span class="pl-k">=</span> builder<span class="pl-k">.</span>buildWriterForCSVInput();
-
- <span class="pl-k">int</span> rows <span class="pl-k">=</span> <span class="pl-c1">5</span>;
- <span class="pl-k">for</span> (<span class="pl-k">int</span> i <span class="pl-k">=</span> <span class="pl-c1">0</span>; i <span class="pl-k"><</span> rows; i<span class="pl-k">++</span>) {
- writer<span class="pl-k">.</span>write(<span class="pl-k">new</span> <span class="pl-smi">String</span>[] { <span class="pl-s"><span class="pl-pds">"</span>robot<span class="pl-pds">"</span></span> <span class="pl-k">+</span> (i <span class="pl-k">%</span> <span class="pl-c1">10</span>), <span class="pl-smi">String</span><span class="pl-k">.</span>valueOf(i) });
- }
- writer<span class="pl-k">.</span>close();
- }
- }</pre></div>
-<h3>
-<a id="example-with-avro-format" class="anchor" href="#example-with-avro-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example with Avro format</h3>
-<div class="highlight highlight-source-java"><pre><span class="pl-k">import</span> <span class="pl-smi">java.io.IOException</span>;
-
-<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException</span>;
-<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.core.metadata.datatype.DataTypes</span>;
-<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.AvroCarbonWriter</span>;
-<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.CarbonWriter</span>;
-<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.Field</span>;
-
-<span class="pl-k">import</span> <span class="pl-smi">org.apache.avro.generic.GenericData</span>;
-<span class="pl-k">import</span> <span class="pl-smi">org.apache.commons.lang.CharEncoding</span>;
-
-<span class="pl-k">import</span> <span class="pl-smi">tech.allegro.schema.json2avro.converter.JsonAvroConverter</span>;
-
-<span class="pl-k">public</span> <span class="pl-k">class</span> <span class="pl-en">TestSdkAvro</span> {
-
- <span class="pl-k">public</span> <span class="pl-k">static</span> <span class="pl-k">void</span> <span class="pl-en">main</span>(<span class="pl-k">String</span>[] <span class="pl-v">args</span>) <span class="pl-k">throws</span> <span class="pl-smi">IOException</span>, <span class="pl-smi">InvalidLoadOptionException</span> {
- testSdkWriter();
- }
-
-
- <span class="pl-k">public</span> <span class="pl-k">static</span> <span class="pl-k">void</span> <span class="pl-en">testSdkWriter</span>() <span class="pl-k">throws</span> <span class="pl-smi">IOException</span>, <span class="pl-smi">InvalidLoadOptionException</span> {
- <span class="pl-smi">String</span> path <span class="pl-k">=</span> <span class="pl-s"><span class="pl-pds">"</span>./AvroCarbonWriterSuiteWriteFiles<span class="pl-pds">"</span></span>;
- <span class="pl-c"><span class="pl-c">//</span> Avro schema</span>
- <span class="pl-smi">String</span> avroSchema <span class="pl-k">=</span>
- <span class="pl-s"><span class="pl-pds">"</span>{<span class="pl-pds">"</span></span> <span class="pl-k">+</span>
- <span class="pl-s"><span class="pl-pds">"</span> <span class="pl-cce">\"</span>type<span class="pl-cce">\"</span> : <span class="pl-cce">\"</span>record<span class="pl-cce">\"</span>,<span class="pl-pds">"</span></span> <span class="pl-k">+</span>
- <span class="pl-s"><span class="pl-pds">"</span> <span class="pl-cce">\"</span>name<span class="pl-cce">\"</span> : <span class="pl-cce">\"</span>Acme<span class="pl-cce">\"</span>,<span class="pl-pds">"</span></span> <span class="pl-k">+</span>
- <span class="pl-s"><span class="pl-pds">"</span> <span class="pl-cce">\"</span>fields<span class="pl-cce">\"</span> : [<span class="pl-pds">"</span></span>
- <span class="pl-k">+</span> <span class="pl-s"><span class="pl-pds">"</span>{ <span class="pl-cce">\"</span>name<span class="pl-cce">\"</span> : <span class="pl-cce">\"</span>fname<span class="pl-cce">\"</span>, <span class="pl-cce">\"</span>type<span class="pl-cce">\"</span> : <span class="pl-cce">\"</span>string<span class="pl-cce">\"</span> },<span class="pl-pds">"</span></span>
- <span class="pl-k">+</span> <span class="pl-s"><span class="pl-pds">"</span>{ <span class="pl-cce">\"</span>name<span class="pl-cce">\"</span> : <span class="pl-cce">\"</span>age<span class="pl-cce">\"</span>, <span class="pl-cce">\"</span>type<span class="pl-cce">\"</span> : <span class="pl-cce">\"</span>int<span class="pl-cce">\"</span> }]<span class="pl-pds">"</span></span> <span class="pl-k">+</span>
- <span class="pl-s"><span class="pl-pds">"</span>}<span class="pl-pds">"</span></span>;
-
- <span class="pl-smi">String</span> json <span class="pl-k">=</span> <span class="pl-s"><span class="pl-pds">"</span>{<span class="pl-cce">\"</span>fname<span class="pl-cce">\"</span>:<span class="pl-cce">\"</span>bob<span class="pl-cce">\"</span>, <span class="pl-cce">\"</span>age<span class="pl-cce">\"</span>:10}<span class="pl-pds">"</span></span>;
-
- <span class="pl-c"><span class="pl-c">//</span> conversion to GenericData.Record</span>
- <span class="pl-smi">JsonAvroConverter</span> converter <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">JsonAvroConverter</span>();
- <span class="pl-smi">GenericData</span><span class="pl-k">.</span><span class="pl-smi">Record</span> record <span class="pl-k">=</span> converter<span class="pl-k">.</span>convertToGenericDataRecord(
- json<span class="pl-k">.</span>getBytes(<span class="pl-smi">CharEncoding</span><span class="pl-c1"><span class="pl-k">.</span>UTF_8</span>), <span class="pl-k">new</span> <span class="pl-smi">org.apache.avro<span class="pl-k">.</span>Schema</span>.<span class="pl-smi">Parser</span>()<span class="pl-k">.</span>parse(avroSchema));
-
- <span class="pl-c"><span class="pl-c">//</span> prepare carbon schema from avro schema </span>
- <span class="pl-smi">org.apache.carbondata.sdk.file<span class="pl-k">.</span>Schema</span> carbonSchema <span class="pl-k">=</span>
- <span class="pl-smi">AvroCarbonWriter</span><span class="pl-k">.</span>getCarbonSchemaFromAvroSchema(avroSchema);
-
- <span class="pl-k">try</span> {
- <span class="pl-smi">CarbonWriter</span> writer <span class="pl-k">=</span> <span class="pl-smi">CarbonWriter</span><span class="pl-k">.</span>builder()
- .withSchema(carbonSchema)
- .outputPath(path)
- .buildWriterForAvroInput();
-
- <span class="pl-k">for</span> (<span class="pl-k">int</span> i <span class="pl-k">=</span> <span class="pl-c1">0</span>; i <span class="pl-k"><</span> <span class="pl-c1">100</span>; i<span class="pl-k">++</span>) {
- writer<span class="pl-k">.</span>write(record);
- }
- writer<span class="pl-k">.</span>close();
- } <span class="pl-k">catch</span> (<span class="pl-smi">Exception</span> e) {
- e<span class="pl-k">.</span>printStackTrace();
- }
- }
-}</pre></div>
-<h2>
-<a id="datatypes-mapping" class="anchor" href="#datatypes-mapping" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Datatypes Mapping</h2>
-<p>Each of SQL data types are mapped into data types of SDK. Following are the mapping:</p>
-<table>
-<thead>
-<tr>
-<th>SQL DataTypes</th>
-<th>Mapped SDK DataTypes</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>BOOLEAN</td>
-<td>DataTypes.BOOLEAN</td>
-</tr>
-<tr>
-<td>SMALLINT</td>
-<td>DataTypes.SHORT</td>
-</tr>
-<tr>
-<td>INTEGER</td>
-<td>DataTypes.INT</td>
-</tr>
-<tr>
-<td>BIGINT</td>
-<td>DataTypes.LONG</td>
-</tr>
-<tr>
-<td>DOUBLE</td>
-<td>DataTypes.DOUBLE</td>
-</tr>
-<tr>
-<td>VARCHAR</td>
-<td>DataTypes.STRING</td>
-</tr>
-<tr>
-<td>DATE</td>
-<td>DataTypes.DATE</td>
-</tr>
-<tr>
-<td>TIMESTAMP</td>
-<td>DataTypes.TIMESTAMP</td>
-</tr>
-<tr>
-<td>STRING</td>
-<td>DataTypes.STRING</td>
-</tr>
-<tr>
-<td>DECIMAL</td>
-<td>DataTypes.createDecimalType(precision, scale)</td>
-</tr>
-</tbody>
-</table>
-<h2>
-<a id="api-list" class="anchor" href="#api-list" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>API List</h2>
-<h3>
-<a id="class-orgapachecarbondatasdkfilecarbonwriterbuilder" class="anchor" href="#class-orgapachecarbondatasdkfilecarbonwriterbuilder" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.CarbonWriterBuilder</h3>
-<pre><code>/**
-* prepares the builder with the schema provided
-* @param schema is instance of Schema
-* This method must be called when building CarbonWriterBuilder
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder withSchema(Schema schema);
-</code></pre>
-<pre><code>/**
-* Sets the output path of the writer builder
-* @param path is the absolute path where output files are written
-* This method must be called when building CarbonWriterBuilder
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder outputPath(String path);
-</code></pre>
-<pre><code>/**
-* If set false, writes the carbondata and carbonindex files in a flat folder structure
-* @param isTransactionalTable is a boolelan value
-* if set to false, then writes the carbondata and carbonindex files
-* in a flat folder structure.
-* if set to true, then writes the carbondata and carbonindex files
-* in segment folder structure..
-* By default set to false.
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder isTransactionalTable(boolean isTransactionalTable);
-</code></pre>
-<pre><code>/**
-* to set the timestamp in the carbondata and carbonindex index files
-* @param UUID is a timestamp to be used in the carbondata and carbonindex index files.
-* By default set to zero.
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder uniqueIdentifier(long UUID);
-</code></pre>
-<pre><code>/**
-* To set the carbondata file size in MB between 1MB-2048MB
-* @param blockSize is size in MB between 1MB to 2048 MB
-* default value is 1024 MB
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder withBlockSize(int blockSize);
-</code></pre>
-<pre><code>/**
-* To set the blocklet size of carbondata file
-* @param blockletSize is blocklet size in MB
-* default value is 64 MB
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder withBlockletSize(int blockletSize);
-</code></pre>
-<pre><code>/**
-* sets the list of columns that needs to be in sorted order
-* @param sortColumns is a string array of columns that needs to be sorted.
-* If it is null or by default all dimensions are selected for sorting
-* If it is empty array, no columns are sorted
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder sortBy(String[] sortColumns);
-</code></pre>
-<pre><code>/**
-* If set, create a schema file in metadata folder.
-* @param persist is a boolean value, If set to true, creates a schema file in metadata folder.
-* By default set to false. will not create metadata folder
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder persistSchemaFile(boolean persist);
-</code></pre>
-<pre><code>/**
-* sets the taskNo for the writer. SDKs concurrently running
-* will set taskNo in order to avoid conflicts in file's name during write.
-* @param taskNo is the TaskNo user wants to specify.
-* by default it is system time in nano seconds.
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder taskNo(String taskNo);
-</code></pre>
-<pre><code>/**
-* To support the load options for sdk writer
-* @param options key,value pair of load options.
-* supported keys values are
-* a. bad_records_logger_enable -- true (write into separate logs), false
-* b. bad_records_action -- FAIL, FORCE, IGNORE, REDIRECT
-* c. bad_record_path -- path
-* d. dateformat -- same as JAVA SimpleDateFormat
-* e. timestampformat -- same as JAVA SimpleDateFormat
-* f. complex_delimiter_level_1 -- value to Split the complexTypeData
-* g. complex_delimiter_level_2 -- value to Split the nested complexTypeData
-* h. quotechar
-* i. escapechar
-*
-* Default values are as follows.
-*
-* a. bad_records_logger_enable -- "false"
-* b. bad_records_action -- "FAIL"
-* c. bad_record_path -- ""
-* d. dateformat -- "" , uses from carbon.properties file
-* e. timestampformat -- "", uses from carbon.properties file
-* f. complex_delimiter_level_1 -- "$"
-* g. complex_delimiter_level_2 -- ":"
-* h. quotechar -- "\""
-* i. escapechar -- "\\"
-*
-* @return updated CarbonWriterBuilder
-*/
-public CarbonWriterBuilder withLoadOptions(Map<String, String> options);
-</code></pre>
-<pre><code>/**
-* Build a {@link CarbonWriter}, which accepts row in CSV format object
-* @return CSVCarbonWriter
-* @throws IOException
-* @throws InvalidLoadOptionException
-*/
-public CarbonWriter buildWriterForCSVInput() throws IOException, InvalidLoadOptionException;
-</code></pre>
-<pre><code>/**
-* Build a {@link CarbonWriter}, which accepts Avro format object
-* @return AvroCarbonWriter
-* @throws IOException
-* @throws InvalidLoadOptionException
-*/
-public CarbonWriter buildWriterForAvroInput() throws IOException, InvalidLoadOptionException;
-</code></pre>
-<h3>
-<a id="class-orgapachecarbondatasdkfilecarbonwriter" class="anchor" href="#class-orgapachecarbondatasdkfilecarbonwriter" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.CarbonWriter</h3>
-<pre><code>/**
-* Write an object to the file, the format of the object depends on the implementation
-* If AvroCarbonWriter, object is of type org.apache.avro.generic.GenericData.Record
-* If CSVCarbonWriter, object is of type String[]
-* Note: This API is not thread safe
-* @param object
-* @throws IOException
-*/
-public abstract void write(Object object) throws IOException;
-</code></pre>
-<pre><code>/**
-* Flush and close the writer
-*/
-public abstract void close() throws IOException;
-</code></pre>
-<pre><code>/**
-* Create a {@link CarbonWriterBuilder} to build a {@link CarbonWriter}
-*/
-public static CarbonWriterBuilder builder() {
-return new CarbonWriterBuilder();
-}
-</code></pre>
-<h3>
-<a id="class-orgapachecarbondatasdkfilefield" class="anchor" href="#class-orgapachecarbondatasdkfilefield" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.Field</h3>
-<pre><code>/**
-* Field Constructor
-* @param name name of the field
-* @param type datatype of field, specified in strings.
-*/
-public Field(String name, String type);
-</code></pre>
-<pre><code>/**
-* Field constructor
-* @param name name of the field
-* @param type datatype of the field of class DataType
-*/
-public Field(String name, DataType type);
-</code></pre>
-<h3>
-<a id="class-orgapachecarbondatasdkfileschema" class="anchor" href="#class-orgapachecarbondatasdkfileschema" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.Schema</h3>
-<pre><code>/**
-* construct a schema with fields
-* @param fields
-*/
-public Schema(Field[] fields);
-</code></pre>
-<pre><code>/**
-* Create a Schema using JSON string, for example:
-* [
-* {"name":"string"},
-* {"age":"int"}
-* ]
-* @param json specified as string
-* @return Schema
-*/
-public static Schema parseJson(String json);
-</code></pre>
-<h3>
-<a id="class-orgapachecarbondatasdkfileavrocarbonwriter" class="anchor" href="#class-orgapachecarbondatasdkfileavrocarbonwriter" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.AvroCarbonWriter</h3>
-<pre><code>/**
-* converts avro schema to carbon schema, required by carbonWriter
-*
-* @param avroSchemaString json formatted avro schema as string
-* @return carbon sdk schema
-*/
-public static org.apache.carbondata.sdk.file.Schema getCarbonSchemaFromAvroSchema(String avroSchemaString);
-</code></pre>
-</div>
-</div>
-</div>
-</div>
-<div class="doc-footer">
- <a href="#top" class="scroll-top">Top</a>
-</div>
-</div>
-</section>
-</div>
-</div>
-</div>
-</section><!-- End systemblock part -->
-<script src="js/custom.js"></script>
-</body>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/security.html
----------------------------------------------------------------------
diff --git a/content/security.html b/content/security.html
index 9168fa3..1699aed 100644
--- a/content/security.html
+++ b/content/security.html
@@ -61,7 +61,7 @@
target="_blank">Release Archive</a></li>
</ul>
</li>
- <li><a href="mainpage.html" class="">Documentation</a></li>
+ <li><a href="documentation.html" class="">Documentation</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
aria-expanded="false">Community <span class="caret"></span></a>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/segment-management-on-carbondata.html
----------------------------------------------------------------------
diff --git a/content/segment-management-on-carbondata.html b/content/segment-management-on-carbondata.html
index 1e6f61d..1d70f24 100644
--- a/content/segment-management-on-carbondata.html
+++ b/content/segment-management-on-carbondata.html
@@ -52,6 +52,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
target="_blank">Apache CarbonData 1.4.1</a></li>
<li>
@@ -179,7 +182,12 @@
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
</div>
- <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a>
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/streaming-guide.html
----------------------------------------------------------------------
diff --git a/content/streaming-guide.html b/content/streaming-guide.html
index 86f0385..e82b84a 100644
--- a/content/streaming-guide.html
+++ b/content/streaming-guide.html
@@ -52,6 +52,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
target="_blank">Apache CarbonData 1.4.1</a></li>
<li>
@@ -179,7 +182,12 @@
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
</div>
- <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a>
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
@@ -360,7 +368,7 @@ streaming table using following DDL.</p>
<p>At the begin of streaming ingestion, the system will try to acquire the table level lock of streaming.lock file. If the system isn't able to acquire the lock of this table, it will throw an InterruptedException.</p>
<h2>
<a id="create-streaming-segment" class="anchor" href="#create-streaming-segment" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create streaming segment</h2>
-<p>The input data of streaming will be ingested into a segment of the CarbonData table, the status of this segment is streaming. CarbonData call it a streaming segment. The "tablestatus" file will record the segment status and data size. The user can use ?SHOW SEGMENTS FOR TABLE tableName? to check segment status.</p>
+<p>The streaming data will be ingested into a separate segment of carbondata table, this segment is termed as streaming segment. The status of this segment will be recorded as "streaming" in "tablestatus" file along with its data size. You can use "SHOW SEGMENTS FOR TABLE tableName" to check segment status.</p>
<p>After the streaming segment reaches the max size, CarbonData will change the segment status to "streaming finish" from "streaming", and create new "streaming" segment to continue to ingest streaming data.</p>
<table>
<thead>
@@ -533,8 +541,9 @@ streaming table using following DDL.</p>
| register TIMESTAMP,
| updated TIMESTAMP
|)
- |STORED BY carbondata
+ |STORED AS carbondata
|TBLPROPERTIES (
+ | 'streaming'='source',
| 'format'='csv',
| 'path'='$csvDataDir'
|)
@@ -553,7 +562,7 @@ streaming table using following DDL.</p>
| register TIMESTAMP,
| updated TIMESTAMP
|)
- |STORED BY carbondata
+ |STORED AS carbondata
|TBLPROPERTIES (
| 'streaming'='true'
|)
@@ -576,7 +585,7 @@ streaming table using following DDL.</p>
sql("SHOW STREAMS [ON TABLE tableName]")
</code></pre>
<p>In above example, two table is created: source and sink. The <code>source</code> table's format is <code>csv</code> and <code>sink</code> table format is <code>carbon</code>. Then a streaming job is created to stream data from source table to sink table.</p>
-<p>These two tables are normal carbon table, they can be queried independently.</p>
+<p>These two tables are normal carbon tables, they can be queried independently.</p>
<h3>
<a id="streaming-job-management" class="anchor" href="#streaming-job-management" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Streaming Job Management</h3>
<p>As above example shown:</p>
@@ -601,18 +610,22 @@ streaming table using following DDL.</p>
name STRING,
age <span class="pl-k">INT</span>
)
-STORED BY carbondata
+STORED <span class="pl-k">AS</span> carbondata
TBLPROPERTIES(
- <span class="pl-s"><span class="pl-pds">'</span>format<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>socket<span class="pl-pds">'</span></span>,
- <span class="pl-s"><span class="pl-pds">'</span>host<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>localhost<span class="pl-pds">'</span></span>,
- <span class="pl-s"><span class="pl-pds">'</span>port<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>8888<span class="pl-pds">'</span></span>
+ <span class="pl-s"><span class="pl-pds">'</span>streaming<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>source<span class="pl-pds">'</span></span>,
+ <span class="pl-s"><span class="pl-pds">'</span>format<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>socket<span class="pl-pds">'</span></span>,
+ <span class="pl-s"><span class="pl-pds">'</span>host<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>localhost<span class="pl-pds">'</span></span>,
+ <span class="pl-s"><span class="pl-pds">'</span>port<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>8888<span class="pl-pds">'</span></span>,
+ <span class="pl-s"><span class="pl-pds">'</span>record_format<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>csv<span class="pl-pds">'</span></span>, <span class="pl-k">//</span> can be csv <span class="pl-k">or</span> json, default is csv
+ <span class="pl-s"><span class="pl-pds">'</span>delimiter<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>|<span class="pl-pds">'</span></span>
)</pre></div>
<p>will translate to</p>
<div class="highlight highlight-source-scala"><pre>spark.readStream
.schema(tableSchema)
.format(<span class="pl-s"><span class="pl-pds">"</span>socket<span class="pl-pds">"</span></span>)
.option(<span class="pl-s"><span class="pl-pds">"</span>host<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>localhost<span class="pl-pds">"</span></span>)
- .option(<span class="pl-s"><span class="pl-pds">"</span>port<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>8888<span class="pl-pds">"</span></span>)</pre></div>
+ .option(<span class="pl-s"><span class="pl-pds">"</span>port<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>8888<span class="pl-pds">"</span></span>)
+ .option(<span class="pl-s"><span class="pl-pds">"</span>delimiter<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>|<span class="pl-pds">"</span></span>)</pre></div>
</li>
<li>
<p>The sink table should have a TBLPROPERTY <code>'streaming'</code> equal to <code>true</code>, indicating it is a streaming table.</p>
@@ -621,9 +634,23 @@ TBLPROPERTIES(
<p>In the given STMPROPERTIES, user must specify <code>'trigger'</code>, its value must be <code>ProcessingTime</code> (In future, other value will be supported). User should also specify interval value for the streaming job.</p>
</li>
<li>
-<p>If the schema specifid in sink table is different from CTAS, the streaming job will fail</p>
+<p>If the schema specified in sink table is different from CTAS, the streaming job will fail</p>
</li>
</ul>
+<p>For Kafka data source, create the source table by:</p>
+<div class="highlight highlight-source-sql"><pre><span class="pl-k">CREATE</span> <span class="pl-k">TABLE</span> <span class="pl-en">source</span>(
+ name STRING,
+ age <span class="pl-k">INT</span>
+)
+STORED <span class="pl-k">AS</span> carbondata
+TBLPROPERTIES(
+ <span class="pl-s"><span class="pl-pds">'</span>streaming<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>source<span class="pl-pds">'</span></span>,
+ <span class="pl-s"><span class="pl-pds">'</span>format<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>kafka<span class="pl-pds">'</span></span>,
+ <span class="pl-s"><span class="pl-pds">'</span>kafka.bootstrap.servers<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>kafkaserver:9092<span class="pl-pds">'</span></span>,
+ <span class="pl-s"><span class="pl-pds">'</span>subscribe<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>test<span class="pl-pds">'</span></span>
+ <span class="pl-s"><span class="pl-pds">'</span>record_format<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>csv<span class="pl-pds">'</span></span>, <span class="pl-k">//</span> can be csv <span class="pl-k">or</span> json, default is csv
+ <span class="pl-s"><span class="pl-pds">'</span>delimiter<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>|<span class="pl-pds">'</span></span>
+)</pre></div>
<h5>
<a id="stop-stream" class="anchor" href="#stop-stream" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>STOP STREAM</h5>
<p>When this is issued, the streaming job will be stopped immediately. It will fail if the jobName specified is not exist.</p>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/supported-data-types-in-carbondata.html
----------------------------------------------------------------------
diff --git a/content/supported-data-types-in-carbondata.html b/content/supported-data-types-in-carbondata.html
index f346052..6c692b8 100644
--- a/content/supported-data-types-in-carbondata.html
+++ b/content/supported-data-types-in-carbondata.html
@@ -52,6 +52,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
target="_blank">Apache CarbonData 1.4.1</a></li>
<li>
@@ -179,7 +182,12 @@
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
</div>
- <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a>
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
@@ -223,7 +231,10 @@
<li>BIGINT</li>
<li>DOUBLE</li>
<li>DECIMAL</li>
+<li>FLOAT</li>
+<li>BYTE</li>
</ul>
+<p><strong>NOTE</strong>: Float and Bytes are only supported for SDK and FileFormat.</p>
</li>
<li>
<p>Date/Time Types</p>
@@ -249,6 +260,8 @@ Please refer to TBLProperties in <a href="./ddl-of-carbondata.html#create-table"
</li>
<li>structs: STRUCT<code><col_name : data_type COMMENT col_comment, ...></code>
</li>
+<li>maps: MAP<code><primitive_type, data_type></code>
+</li>
</ul>
<p><strong>NOTE</strong>: Only 2 level complex type schema is supported for now.</p>
</li>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/6ad7599a/content/timeseries-datamap-guide.html
----------------------------------------------------------------------
diff --git a/content/timeseries-datamap-guide.html b/content/timeseries-datamap-guide.html
index 73a4580..d34a013 100644
--- a/content/timeseries-datamap-guide.html
+++ b/content/timeseries-datamap-guide.html
@@ -52,6 +52,9 @@
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
+ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.0/"
+ target="_blank">Apache CarbonData 1.5.0</a></li>
+ <li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"
target="_blank">Apache CarbonData 1.4.1</a></li>
<li>
@@ -179,7 +182,12 @@
<a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a>
</div>
- <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a>
+ <div class="nav__item nav__item__with__subs">
+ <a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
+ <a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
+ <a class="nav__item nav__sub__item" href="./CSDK-guide.html">C++ SDK</a>
+ </div>
+
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>