You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/02/03 09:26:52 UTC
svn commit: r1728267 - in /kylin/site: ./ development/ images/develop/
Author: mahongbin
Date: Wed Feb 3 08:26:52 2016
New Revision: 1728267
URL: http://svn.apache.org/viewvc?rev=1728267&view=rev
Log:
UPDATE MESSAGE
Added:
kylin/site/development/streaming_cubing.html
kylin/site/images/develop/streaming.png (with props)
Modified:
kylin/site/development/about_temp_files.html
kylin/site/development/dev_env.html
kylin/site/development/howto_contribute.html
kylin/site/development/howto_docs.html
kylin/site/development/howto_package.html
kylin/site/development/howto_release.html
kylin/site/development/howto_test.html
kylin/site/development/index.html
kylin/site/development/new_metadata.html
kylin/site/development/web_tech.html
kylin/site/feed.xml
Modified: kylin/site/development/about_temp_files.html
URL: http://svn.apache.org/viewvc/kylin/site/development/about_temp_files.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/about_temp_files.html (original)
+++ kylin/site/development/about_temp_files.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
Modified: kylin/site/development/dev_env.html
URL: http://svn.apache.org/viewvc/kylin/site/development/dev_env.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/dev_env.html (original)
+++ kylin/site/development/dev_env.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_contribute.html
URL: http://svn.apache.org/viewvc/kylin/site/development/howto_contribute.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_contribute.html (original)
+++ kylin/site/development/howto_contribute.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_docs.html
URL: http://svn.apache.org/viewvc/kylin/site/development/howto_docs.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_docs.html (original)
+++ kylin/site/development/howto_docs.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_package.html
URL: http://svn.apache.org/viewvc/kylin/site/development/howto_package.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_package.html (original)
+++ kylin/site/development/howto_package.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_release.html
URL: http://svn.apache.org/viewvc/kylin/site/development/howto_release.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_release.html (original)
+++ kylin/site/development/howto_release.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_test.html
URL: http://svn.apache.org/viewvc/kylin/site/development/howto_test.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_test.html (original)
+++ kylin/site/development/howto_test.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
@@ -441,7 +476,7 @@
<article class="post-content" >
<p>In general, there should be unit tests to cover individual classes; there must be integration test to cover end-to-end scenarios like build, merge, and query. Unit test must run independently (does not require an external sandbox).</p>
-<h2 id="x-branches">2.x branches</h2>
+<h2 id="test-2x-branches">Test 2.x branches</h2>
<ul>
<li><code class="highlighter-rouge">mvn test</code> to run unit tests, which has a limited test coverage.
@@ -453,15 +488,15 @@
</li>
<li><code class="highlighter-rouge">dev-support/test_all_against_hdp_2_2_4_2_2.sh</code> to run integration tests, which has the best test coverage.
<ul>
- <li>Integration tests <strong>must run on a Hadoop sandbox</strong>. Make sure all changes you want to test are avaiable on sandbox.</li>
+ <li>Integration tests <strong>better be run on a Hadoop sandbox</strong>. We suggest to checkout a copy of code in your sandbox and direct run the test_all_against_hdp_2_2_4_2_2.sh in it. If you donât want to put codes on sandbox, refer to <strong>More on 2.x UT/IT separation</strong></li>
<li>As the name indicates, the script is only for hdp 2.2.4.2, but you get the idea of how integration test run from it.</li>
<li>The integration tests start from generate random data, then build cube, merge cube, and finally query the result and compare to H2 DB.</li>
- <li>The integration tests take a few hours to complete.</li>
+ <li>The integration tests take one to two hours to complete.</li>
</ul>
</li>
</ul>
-<h2 id="x-branches-1">1.x branches</h2>
+<h2 id="test-1x-branches">Test 1.x branches</h2>
<ul>
<li><code class="highlighter-rouge">mvn test</code> to run unit tests, which has a limited test coverage.
@@ -480,6 +515,32 @@
</li>
</ul>
+<h2 id="more-on-2x-utit-separation">More on 2.x UT/IT separation</h2>
+
+<p>From Kylin 2.0 you can run UT(Unit test), environment cube provision and IT(Integration test) separately. <br />
+Running <code class="highlighter-rouge">mvn verify -Dhdp.version=2.2.4.2-2</code> (assume youâre on your sandbox) is all you need to run a complete all the test suites.</p>
+
+<p>It will execute the following steps sequentially:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>1. Build Artifacts
+2. Run all UTs (takes few minutes)
+3. Provision cubes on the sandbox environment for IT uasge (takes 1~2 hours)
+4. Run all ITs (takes few tens of minutes)
+5. verify jar stuff
+</code></pre>
+</div>
+
+<p>If your code change is minor and it merely requires running UT, use: <br />
+<code class="highlighter-rouge">mvn test</code><br />
+If your sandbox is already provisioned and your code change will not affect the result of sandbox provision, (and you donât want to wait hours of provision) just run the following commands to separately run UT and IT: <br />
+<code class="highlighter-rouge">mvn test</code><br />
+<code class="highlighter-rouge">mvn failsafe:integration-test</code></p>
+
+<h3 id="cube-provision">Cube Provision</h3>
+
+<p>Environment cube provision is indeed running kylin cubing jobs to prepare example cubes in the sandbox. These prepared cubes will be used by the ITs. Currently provision step is bound with the maven pre-integration-test phase, and it contains running BuildCubeWithEngine (HBase required), BuildCubeWithStream(Kafka required) and BuildIIWithStream(Kafka Required). You can run the mvn commands on you sandbox or your develop computer. For the latter case you need to set kylin.job.run.as.remote.cmd=true in <strong>$KYLIN_HOME/examples/test_case_data/sandbox/kylin.properties</strong>. <br />
+Try appending <code class="highlighter-rouge">-DfastBuildMode=true</code> to mvn verify command to speed up provision by skipping incremental cubing.</p>
+
<h2 id="more-on-1x-mini-cluster">More on 1.x Mini Cluster</h2>
<p>Kylin 1.x used to move as many as possible unit test cases from sandbox to HBase mini cluster (not any more in 2.x), so that user can run tests easily in local without a hadoop sandbox. Two maven profiles are created in the root pom.xml, âdefaultâ and âsandboxâ. The default profile will startup a HBase Mini Cluster to prepare the test data and run the unit tests (the test cases that are not supported by Mini cluster have been added in the âexcludeâ list). If you want to keep using Sandbox to run test, just run <code class="highlighter-rouge">mvn test -P sandbox</code></p>
@@ -494,6 +555,7 @@
</ul>
<h3 id="to-ensure-mini-cluster-can-run-successfully-you-need">To ensure Mini cluster can run successfully, you need</h3>
+
<ul>
<li>Make sure JAVA_HOME is properly set</li>
</ul>
Modified: kylin/site/development/index.html
URL: http://svn.apache.org/viewvc/kylin/site/development/index.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/index.html (original)
+++ kylin/site/development/index.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
Modified: kylin/site/development/new_metadata.html
URL: http://svn.apache.org/viewvc/kylin/site/development/new_metadata.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/new_metadata.html (original)
+++ kylin/site/development/new_metadata.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
Added: kylin/site/development/streaming_cubing.html
URL: http://svn.apache.org/viewvc/kylin/site/development/streaming_cubing.html?rev=1728267&view=auto
==============================================================================
--- kylin/site/development/streaming_cubing.html (added)
+++ kylin/site/development/streaming_cubing.html Wed Feb 3 08:26:52 2016
@@ -0,0 +1,551 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<!doctype html>
+<html>
+ <!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+
+ <title>Apache Kylin | Streaming cubing (Prototype)</title>
+ <meta name="description" content="Apache Kylin Home">
+ <meta name="author" content="Apache Kylin">
+ <link rel="shortcut icon" href="fav.png" type="image/png">
+
+
+
+<link rel="stylesheet" href="/assets/css/animate.css">
+<!-- Bootstrap -->
+<link rel="stylesheet" href="/assets/css/bootstrap.min.css">
+
+<!-- Fonts -->
+<!-- <link rel="stylesheet" href="http://fonts.googleapis.com/css?family=Alice|Open+Sans:400,300,700"> -->
+
+<!-- Icons -->
+<link rel="stylesheet" href="/assets/css/font-awesome.min.css">
+
+ <!-- Custom styles -->
+ <link rel="stylesheet" href="/assets/css/styles.css">
+ <link rel="stylesheet" href="/assets/css/docs.css">
+ <link rel="stylesheet" href="/assets/css/pygments.css">
+
+ <link rel="canonical" href="http://kylin.apache.org/development/streaming_cubing.html">
+ <link rel="alternate" type="application/rss+xml" title="Apache Kylin" href="http://kylin.apache.org/feed.xml" />
+
+<!--[if lt IE 9]> <script src="assets/js/html5shiv.js"></script> <![endif]-->
+<script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ //oringal tracker for kylin.io
+ ga('create', 'UA-55534813-1', 'auto');
+ //new tracker for kylin.apache.org
+ ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'});
+
+ ga('send', 'pageview');
+ ga('toplevel.send', 'pageview');
+
+
+</script>
+<script type="text/javascript" src="/assets/js/jquery-1.9.1.min.js"></script>
+<script type="text/javascript" src="/assets/js/nside.js"></script> </script>
+<script type="text/javascript" src="/assets/js/nnav.js"></script> </script>
+</head>
+
+ <body>
+ <!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<header id="header" >
+
+ <div id="head" class="parallax" parallax-speed="3" >
+ <div id="logo" class="text-center"> <img class="img-circle" id="circlelogo" src="/assets/images/kylin_logo.jpg"> <span class="title" >Apache Kylinâ¢</span> <span class="tagline">Extreme OLAP Engine for Big Data</span>
+ </div>
+ </div>
+
+
+ <!-- Main Menu -->
+ <nav class="navbar navbar-default" role="navigation" id="nav-wrapper">
+ <div class="container-fluid" id="nav">
+ <!--
+ <img class="img-circle" width="40px" height="40px" id="circlelogo" src="/assets/images/kylin_logo.jpg">
+ -->
+ <!-- Brand and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
+ </div>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="/">Home</a></li>
+ <li><a href="/docs" >Docs</a></li>
+ <li><a href="/download">Download</li>
+ <li><a href="/community" >Community</a></li>
+ <li><a href="/development" >Development</a></li>
+ <li><a href="/blog">Blog</li>
+ <li><a href="/cn" >ä¸æç</a></li>
+ <li><a href="https://twitter.com/apachekylin" target="_blank" class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" ></a></li>
+ <li><a href="https://github.com/apache/kylin" target="_blank" class="fa fa-github-alt fa-lg" title="Github: apache/kylin" ></a></li>
+ <li><a href="https://www.facebook.com/kylinio" target="_blank" class="fa fa-facebook fa-lg" title="Facebook: kylin.io" ></a></li>
+ </ul>
+ </div><!-- /.navbar-collapse -->
+ </div><!-- /.container-fluid -->
+</nav>
+ </header>
+
+
+ <div class="container">
+ <div class="row">
+ <!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<div class="col-md-3 col-lg-3 col-xs-4 aside1 visible-md visible-lg" id="nside1" style=" padding-top: 2em">
+ <ul class="nav nav-pills nav-stacked">
+
+ <li><a href="#titleDevelopment Guide" data-toggle="collapse" id="navtitle">Development Guide</a></li>
+ <div class="collapse in">
+ <div class="list-group" id="list1">
+ <ul style="list-style-type:disc">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/index.html" class="list-group-item-lay pjaxlink" id="navlist">Development Quick Guide</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/dev_env.html" class="list-group-item-lay pjaxlink" id="navlist">Setup Development Env</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_test.html" class="list-group-item-lay pjaxlink" id="navlist">How to Test</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_contribute.html" class="list-group-item-lay pjaxlink" id="navlist">How to Contribute</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_docs.html" class="list-group-item-lay pjaxlink" id="navlist">How to Write Document</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_package.html" class="list-group-item-lay pjaxlink" id="navlist">How to Build Binary Package</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_release.html" class="list-group-item-lay pjaxlink" id="navlist">How to Making a Release</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/new_metadata.html" class="list-group-item-lay pjaxlink" id="navlist">New Metadata Model</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/about_temp_files.html" class="list-group-item-lay pjaxlink" id="navlist">About Temp Files</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
+
+
+
+ <ul>
+ </div>
+</div>
+
+
+ </ul>
+</div>
+ <div class="col-md-9 col-lg-9 col-xs-14 aside2">
+ <div id="container">
+ <div id="pjax">
+ <h1 class="post-title">Streaming cubing (Prototype)</h1>
+ <article class="post-content" >
+ <p>One of the most important features in 2.x branches is streaming cubing which enables OLAP analysis on streaming data. Streaming cubing delivers faster insights on the data to help more promptly business decisions. Even though there are already many real time analysis tools in open source community, Kylin Streaming cubing still differs from them in multiple angles:</p>
+
+<p>Firstly, Kylin Streaming Cubing aligns with Kylin traditional cubing to provided unified, ANSI SQL interface. Actually Kylin Streaming shares the storage engine and query engine with traditional Kylin cubes, so in theory all of the optimization techniques to save storage and speed up query performance can also be applied on streaming cubes. Besides, all the supported aggregations/filters/UDFs still work for streaming cubes. By unifying the storage engine and query engine we also get freed from double amount of maintaince work.</p>
+
+<p>Secondly, Kylin Streaming Cubing does not require large amount of memory to store real time data, nor does it attempts to provide truly âreal timeâ analysis. By our customer survey we found that minutes of visualization latency is acceptable for OLAP analysts. So our streaming cubing adopts the micro batch approach. Incoming streaming data are partitioned into different time windows and we build a micro batch for each time window. The cube output for each micro batch is directly saved to HBase. The query engine goes to HBase for data retrieving instead of the data ingestion server. The benefit of such design is that we donât have to maintain large amount of in-memory index which could easily require tens of gigabytes of memory. In the future Kylin might need to consider truly âreal timeâ support, too.</p>
+
+<p>Thirdly, Kylin Streaming Cubing data will be persistent and gradually be converted to traditional cubes, thus customers can still query âcold dataâ without any compromise on performance. As discussed above the output of streaming cubing is directly saved to HBase as a new segment. The traditional job engine will be notified of the new segment and take over to schedule merge jobs when then segments accumulates. Day after day the segments of the streaming cube got merged and become a very large traditional cube.</p>
+
+<p><img src="/images/develop/streaming.png" alt="Kylin Streaming Framework Architecture" /></p>
+
+<p>With the major difference in mind we will introduce the modules for Kylin Streaming cubing. Kylin Streaming cubing consist of three major parts:</p>
+
+<ul>
+ <li>Streaming Input to retrieve data from a replayable data queue (currently it is Kafka) within given time window. Streaming Input is also responsible for primary data cleaning and normalization. By default Kylin Streaming provides a default implementation to parse the messages from the source queue. Customers can choose to configure the parser or provide a brand new one based on their requirements.</li>
+ <li>Streaming Batch Ingestion to ingest the incoming data batch and transform it into a micro cube. Thanks to the latest Kylin In-memory cubing technology, this step is now times faster and space-saving than previous. The micro cube is directly saved to HBase.</li>
+ <li>Job Scheduling Module to trigger Streaming Batch Ingestion. Kylin does not put too much efforts in job scheduling, streaming cubing is not a exception. Currently we provided a simple implementation based on Linux Crontab.</li>
+</ul>
+
+<p>Weâll publish more detailed documents on how to use Kylin Streaming soon. In latest 2.x branch we are also working on more complicated load balancing schemes for streaming cubing. Please stay tuned.</p>
+
+
+ </article>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<footer id="underfooter">
+ <div class="container">
+ <div class="row">
+ <div class="col-md-12 widget">
+ <div class="widget-body" style="text-align:center">
+ <a href="http://www.apache.org">
+ <img id="asf-logo" alt="Apache Software Foundation" src="/assets/images/feather-small.gif">
+ </a>
+
+ <div>
+ The contents of this website are © 2015 Apache Software Foundation under the terms of the <a
+ href="http://www.apache.org/licenses/LICENSE-2.0"> Apache License v2 </a>. Apache Kylin and
+ its logo are trademarks of the Apache Software Foundation.
+ </div>
+
+ </div>
+ </div>
+ </div>
+ <!-- /row of widgets -->
+
+ </div>
+ <div></div>
+
+</footer>
+
+ <script src="/assets/js/jquery-1.9.1.min.js"></script>
+ <script src="/assets/js/bootstrap.min.js"></script>
+ <script src="/assets/js/main.js"></script>
+ </body>
+</html>
Modified: kylin/site/development/web_tech.html
URL: http://svn.apache.org/viewvc/kylin/site/development/web_tech.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/web_tech.html (original)
+++ kylin/site/development/web_tech.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html" class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing (Prototype)</a></li>
+
+
+
Modified: kylin/site/feed.xml
URL: http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/feed.xml (original)
+++ kylin/site/feed.xml Wed Feb 3 08:26:52 2016
@@ -19,8 +19,8 @@
<description>Apache Kylin Home</description>
<link>http://kylin.apache.org/</link>
<atom:link href="http://kylin.apache.org/feed.xml" rel="self" type="application/rss+xml"/>
- <pubDate>Mon, 01 Feb 2016 06:48:24 -0800</pubDate>
- <lastBuildDate>Mon, 01 Feb 2016 06:48:24 -0800</lastBuildDate>
+ <pubDate>Wed, 03 Feb 2016 00:25:14 -0800</pubDate>
+ <lastBuildDate>Wed, 03 Feb 2016 00:25:14 -0800</lastBuildDate>
<generator>Jekyll v2.5.3</generator>
<item>
Added: kylin/site/images/develop/streaming.png
URL: http://svn.apache.org/viewvc/kylin/site/images/develop/streaming.png?rev=1728267&view=auto
==============================================================================
Binary file - no diff available.
Propchange: kylin/site/images/develop/streaming.png
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream