You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/21 01:27:06 UTC
[01/50] [abbrv] incubator-kylin git commit: Update unit tests
Repository: incubator-kylin
Updated Branches:
refs/heads/streaming-localdict af153c42f -> cc1fed44d
Update unit tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0170668f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0170668f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0170668f
Branch: refs/heads/streaming-localdict
Commit: 0170668f0e35fdc9d4826af08e30323c692f4144
Parents: 4a3278c
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Mar 18 11:34:06 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Mar 18 11:34:06 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java | 3 +++
.../org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java | 4 ++--
.../java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java | 4 ++--
.../kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java | 4 ++--
pom.xml | 4 ++++
5 files changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0170668f/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
index 15db7da..a1d3569 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
@@ -42,6 +42,9 @@ public class BaseCuboidJobTest extends LocalFileMetadataTestCase {
@Before
public void setup() throws Exception {
conf = new Configuration();
+ conf.set("fs.default.name", "file:///");
+ conf.set("mapred.job.tracker", "local");
+
// for local runner out-of-memory issue
conf.set("mapreduce.task.io.sort.mb", "10");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0170668f/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
index 640c4d2..aae54b4 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
@@ -42,8 +42,8 @@ public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
@Before
public void setup() throws Exception {
conf = new Configuration();
- // conf.set("fs.default.name", "file:///");
- // conf.set("mapred.job.tracker", "local");
+ conf.set("fs.default.name", "file:///");
+ conf.set("mapred.job.tracker", "local");
// for local runner out-of-memory issue
conf.set("mapreduce.task.io.sort.mb", "10");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0170668f/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
index 0e6fc02..c8a3d51 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
@@ -42,8 +42,8 @@ public class NDCuboidJobTest extends LocalFileMetadataTestCase {
@Before
public void setup() throws Exception {
conf = new Configuration();
- // conf.set("fs.default.name", "file:///");
- // conf.set("mapred.job.tracker", "local");
+ conf.set("fs.default.name", "file:///");
+ conf.set("mapred.job.tracker", "local");
// for local runner out-of-memory issue
conf.set("mapreduce.task.io.sort.mb", "10");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0170668f/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java
index 833a8b3..b3bcc30 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java
@@ -42,8 +42,8 @@ public class RangeKeyDistributionJobTest extends LocalFileMetadataTestCase {
@Before
public void setup() throws Exception {
conf = new Configuration();
- // conf.set("fs.default.name", "file:///");
- // conf.set("mapred.job.tracker", "local");
+ conf.set("fs.default.name", "file:///");
+ conf.set("mapred.job.tracker", "local");
// for local runner out-of-memory issue
conf.set("mapreduce.task.io.sort.mb", "10");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0170668f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 841d75b..7d3fa1b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -621,6 +621,10 @@
<value>true</value>
</property>
<property>
+ <name>hdp.version</name>
+ <value>2.2.0.0-2041</value>
+ </property>
+ <property>
<name>buildCubeUsingProvidedData</name>
<value>false</value>
</property>
[30/50] [abbrv] incubator-kylin git commit: KYLIN-650 category
resting pages to different topics
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5acdcd23/docs/Test/Run Kylin test case with HBase Mini Cluster.md
----------------------------------------------------------------------
diff --git a/docs/Test/Run Kylin test case with HBase Mini Cluster.md b/docs/Test/Run Kylin test case with HBase Mini Cluster.md
new file mode 100644
index 0000000..6088eb9
--- /dev/null
+++ b/docs/Test/Run Kylin test case with HBase Mini Cluster.md
@@ -0,0 +1,15 @@
+## Run test case with HBase Mini Cluster
+
+Kylin is moving as many as possible unit test cases from sandbox to HBase mini cluster, so that user can run tests easily in local without a hadoop sandbox; Two maven profiles are created in the root pom.xml, "default" and "sandbox"; The default profile will startup a HBase Mini Cluster to prepare the test data and run the unit tests (the test cases that are not supported by Mini cluster have been added in the "exclude" list); If you want to keep using Sandbox to run test, just run:
+mvn test -P sandbox
+
+
+### When use the "default" profile, Kylin will:
+
+* Startup a HBase minicluster and update KylinConfig with the dynamic HBase configurations;
+* Create Kylin metadata tables and import six example cube tables;
+* Import the hbase data from a tar ball from local: examples/test_case_data/minicluster/hbase-export.tar.gz (the hbase-export.tar.gz will be updated on complete of running BuildCubeWithEngineTest）
+* After all test cases be completed, shutdown minicluster and cleanup KylinConfig cache;
+
+### To ensure Mini cluster can run successfully, you need:
+* Make sure JAVA_HOME is properly set;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5acdcd23/docs/Tutorial/How to use kylin remote jdbc driver.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/How to use kylin remote jdbc driver.md b/docs/Tutorial/How to use kylin remote jdbc driver.md
new file mode 100644
index 0000000..45de447
--- /dev/null
+++ b/docs/Tutorial/How to use kylin remote jdbc driver.md
@@ -0,0 +1,83 @@
+### Authentication
+Build on kylin authentication restful service. Supported parameters:
+* user : username
+* password : password
+* ssl: true/false. Default be false; If true, all the services call will use https.
+
+### Connection URL format:
+```
+jdbc:kylin://<hostname>:<port>/<kylin_project_name>
+```
+* If "ssl" = true, the "port" should be Kylin server's HTTPS port;
+* If "port" is not specified, the driver will use default port: HTTP 80, HTTPS 443;
+* The "kylin_project_name" must be specified and user need ensure it exists in Kylin server;
+
+### 1. Query with Statement
+```
+ Driver driver = (Driver) Class.forName("org.apache.kylin.jdbc.Driver").newInstance();
+
+ Properties info = new Properties();
+ info.put("user", "ADMIN");
+ info.put("password", "KYLIN");
+ Connection conn = driver.connect("jdbc:kylin://localhost:7070/kylin_project_name", info);
+ Statement state = conn.createStatement();
+ ResultSet resultSet = state.executeQuery("select * from test_table");
+
+ while (resultSet.next()) {
+ assertEquals("foo", resultSet.getString(1));
+ assertEquals("bar", resultSet.getString(2));
+ assertEquals("tool", resultSet.getString(3));
+ }
+```
+
+### 2. Query with PreparedStatement
+Supported prepared statement parameters:
+* setString
+* setInt
+* setShort
+* setLong
+* setFloat
+* setDouble
+* setBoolean
+* setByte
+* setDate
+* setTime
+* setTimestamp
+
+```
+ Driver driver = (Driver) Class.forName("org.apache.kylin.jdbc.Driver").newInstance();
+ Properties info = new Properties();
+ info.put("user", "ADMIN");
+ info.put("password", "KYLIN");
+ Connection conn = driver.connect("jdbc:kylin://localhost:7070/kylin_project_name", info);
+ PreparedStatement state = conn.prepareStatement("select * from test_table where id=?");
+ state.setInt(1, 10);
+ ResultSet resultSet = state.executeQuery();
+
+ while (resultSet.next()) {
+ assertEquals("foo", resultSet.getString(1));
+ assertEquals("bar", resultSet.getString(2));
+ assertEquals("tool", resultSet.getString(3));
+ }
+```
+
+### 3. Get query result metadata
+Kylin jdbc driver supports metadata list methods:
+List catalog, schema, table and column with sql pattern filters(such as %).
+
+```
+ Driver driver = (Driver) Class.forName("org.apache.kylin.jdbc.Driver").newInstance();
+ Properties info = new Properties();
+ info.put("user", "ADMIN");
+ info.put("password", "KYLIN");
+ Connection conn = driver.connect("jdbc:kylin://localhost:7070/kylin_project_name", info);
+ Statement state = conn.createStatement();
+ ResultSet resultSet = state.executeQuery("select * from test_table");
+
+ ResultSet tables = conn.getMetaData().getTables(null, null, "dummy", null);
+ while (tables.next()) {
+ for (int i = 0; i < 10; i++) {
+ assertEquals("dummy", tables.getString(i + 1));
+ }
+ }
+```
[07/50] [abbrv] incubator-kylin git commit: KYLIN-625,
add minimal inverted index
Posted by li...@apache.org.
KYLIN-625, add minimal inverted index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/05c2d096
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/05c2d096
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/05c2d096
Branch: refs/heads/streaming-localdict
Commit: 05c2d096868f468c361225301901ae0631b008dc
Parents: abca45d
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Mar 18 18:22:45 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Mar 18 18:22:45 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ByteArray.java | 6 +
.../apache/kylin/storage/gridtable/GTInfo.java | 57 +++++-
.../storage/gridtable/GTInvertedIndex.java | 201 +++++++++++++++++++
.../gridtable/GTInvertedIndexOfColumn.java | 116 +++++++++++
.../kylin/storage/gridtable/GTRawScanner.java | 16 +-
.../kylin/storage/gridtable/GTRecord.java | 12 +-
.../storage/gridtable/GTRowBlockIndex.java | 5 -
.../storage/gridtable/GTSampleCodeSystem.java | 8 +-
.../apache/kylin/storage/gridtable/GTUtil.java | 23 +++
.../storage/gridtable/GTInvertedIndexTest.java | 165 +++++++++++++++
.../kylin/storage/gridtable/GridTableTest.java | 9 +-
11 files changed, 581 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index 8c9554a..3f096c6 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -82,6 +82,12 @@ public class ByteArray implements Comparable<ByteArray> {
public void setLength(int length) {
this.length = length;
}
+
+ public ByteArray copy() {
+ ByteArray copy = new ByteArray(length);
+ copy.copyFrom(this);
+ return copy;
+ }
public void copyFrom(ByteArray other) {
System.arraycopy(other.array(), other.offset, data, offset, other.length);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
index 36d3a47..97cac87 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -6,9 +6,10 @@ import java.util.Iterator;
import java.util.LinkedList;
import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.TblColRef;
public class GTInfo {
-
+
public static Builder builder() {
return new Builder();
}
@@ -20,6 +21,8 @@ public class GTInfo {
int nColumns;
DataType[] colTypes;
BitSet colAll;
+ BitSet colPreferIndex;
+ transient TblColRef[] colRefs;
// grid info
BitSet primaryKey; // columns sorted and unique
@@ -42,6 +45,34 @@ public class GTInfo {
return rowBlockSize > 0;
}
+ public int getRowBlockSize() {
+ return rowBlockSize;
+ }
+
+ public BitSet selectColumnBlocks(BitSet columns) {
+ if (columns == null)
+ columns = colAll;
+
+ BitSet result = new BitSet();
+ for (int i = 0; i < colBlocks.length; i++) {
+ BitSet cb = colBlocks[i];
+ if (cb.intersects(columns)) {
+ result.set(i);
+ }
+ }
+ return result;
+ }
+
+ public TblColRef colRef(int i) {
+ if (colRefs == null) {
+ colRefs = new TblColRef[nColumns];
+ }
+ if (colRefs[i] == null) {
+ colRefs[i] = GTUtil.tblColRef(i, colTypes[i].toString());
+ }
+ return colRefs[i];
+ }
+
void validate() {
if (codeSystem == null)
@@ -51,7 +82,7 @@ public class GTInfo {
throw new IllegalStateException();
codeSystem.init(this);
-
+
validateColumnBlocks();
}
@@ -60,7 +91,10 @@ public class GTInfo {
colAll.flip(0, nColumns);
colBlocksAll = new BitSet();
colBlocksAll.flip(0, colBlocks.length);
-
+
+ if (colPreferIndex == null)
+ colPreferIndex = new BitSet();
+
// column blocks must not overlap
for (int i = 0; i < colBlocks.length; i++) {
for (int j = i + 1; j < colBlocks.length; j++) {
@@ -68,7 +102,7 @@ public class GTInfo {
throw new IllegalStateException();
}
}
-
+
// column block must cover all columns
BitSet merge = new BitSet();
for (int i = 0; i < colBlocks.length; i++) {
@@ -120,7 +154,7 @@ public class GTInfo {
}
return this;
}
-
+
/** required */
public Builder setPrimaryKey(BitSet primaryKey) {
info.primaryKey = primaryKey;
@@ -138,22 +172,29 @@ public class GTInfo {
info.colBlocks = columnBlocks;
return this;
}
-
+
/** optional */
public Builder enableRowBlock(int rowBlockSize) {
info.rowBlockSize = rowBlockSize;
return this;
}
-
+
/** optional */
public Builder enableSharding(int nShards) {
info.nShards = nShards;
return this;
}
-
+
+ /** optional */
+ public Builder setColumnPreferIndex(BitSet colPreferIndex) {
+ info.colPreferIndex = colPreferIndex;
+ return this;
+ }
+
public GTInfo build() {
info.validate();
return info;
}
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
new file mode 100644
index 0000000..3c8d862
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
@@ -0,0 +1,201 @@
+package org.apache.kylin.storage.gridtable;
+
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.util.BitSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+
+/**
+ * A thread-safe inverted index of row blocks in memory.
+ *
+ * Note function not() must return all blocks, because index only know what block contains a value,
+ * but not sure what block does not contain a value.
+ *
+ * @author yangli9
+ */
+public class GTInvertedIndex {
+
+ private final GTInfo info;
+ private final BitSet colPreferIndex;
+ private final BitSet colBlocks;
+ private final GTInvertedIndexOfColumn[] index; // for each column
+
+ private volatile int nIndexedBlocks;
+
+ public GTInvertedIndex(GTInfo info) {
+ this.info = info;
+ this.colPreferIndex = info.colPreferIndex;
+ this.colBlocks = info.selectColumnBlocks(colPreferIndex);
+
+ index = new GTInvertedIndexOfColumn[info.nColumns];
+ for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
+ index[i] = new GTInvertedIndexOfColumn(info.codeSystem.getFilterCodeSystem());
+ }
+ }
+
+ public void add(GTRowBlock block) {
+
+ @SuppressWarnings("unchecked")
+ Set<ByteArray>[] distinctValues = new Set[info.nColumns];
+ for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
+ distinctValues[i] = new HashSet<ByteArray>();
+ }
+
+ GTRowBlock.Reader reader = block.getReader(colBlocks);
+ GTRecord record = new GTRecord(info);
+ while (reader.hasNext()) {
+ reader.fetchNext(record);
+ for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
+ distinctValues[i].add(record.get(i));
+ }
+ }
+
+ for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
+ index[i].add(distinctValues[i], block.sequenceId());
+ }
+
+ nIndexedBlocks = Math.max(nIndexedBlocks, block.seqId + 1);
+ }
+
+ public ConciseSet filter(TupleFilter filter) {
+ return filter(filter, nIndexedBlocks);
+ }
+
+ public ConciseSet filter(TupleFilter filter, int totalBlocks) {
+ // number of indexed blocks may increase as we do evaluation
+ int indexedBlocks = nIndexedBlocks;
+
+ Evaluator evaluator = new Evaluator(indexedBlocks);
+ ConciseSet r = evaluator.evaluate(filter);
+
+ // add blocks that have not been indexed
+ for (int i = indexedBlocks; i < totalBlocks; i++) {
+ r.add(i);
+ }
+
+ return r;
+ }
+
+ private class Evaluator {
+ private int indexedBlocks;
+
+ Evaluator(int indexedBlocks) {
+ this.indexedBlocks = indexedBlocks;
+ }
+
+ public ConciseSet evaluate(TupleFilter filter) {
+ if (filter == null) {
+ return all();
+ }
+
+ if (filter instanceof LogicalTupleFilter)
+ return evalLogical((LogicalTupleFilter) filter);
+
+ if (filter instanceof CompareTupleFilter)
+ return evalCompare((CompareTupleFilter) filter);
+
+ // unable to evaluate
+ return all();
+ }
+
+ @SuppressWarnings("unchecked")
+ private ConciseSet evalCompare(CompareTupleFilter filter) {
+ int col = col(filter);
+ if (index[col] == null)
+ return all();
+
+ switch (filter.getOperator()) {
+ case ISNULL:
+ return index[col].getNull();
+ case ISNOTNULL:
+ return all();
+ case EQ:
+ return index[col].getEquals((ByteArray) filter.getFirstValue());
+ case NEQ:
+ return all();
+ case IN:
+ return index[col].getIn((Iterable<ByteArray>) filter.getValues());
+ case NOTIN:
+ return all();
+ case LT:
+ return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), false);
+ case LTE:
+ return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), true);
+ case GT:
+ return index[col].getRange((ByteArray) filter.getFirstValue(), false, null, false);
+ case GTE:
+ return index[col].getRange((ByteArray) filter.getFirstValue(), true, null, false);
+ default:
+ throw new IllegalStateException("Unsupported operator " + filter.getOperator());
+ }
+ }
+
+ private ConciseSet evalLogical(LogicalTupleFilter filter) {
+ List<? extends TupleFilter> children = filter.getChildren();
+
+ switch (filter.getOperator()) {
+ case AND:
+ return evalLogicalAnd(children);
+ case OR:
+ return evalLogicalOr(children);
+ case NOT:
+ return evalLogicalNot(children);
+ default:
+ throw new IllegalStateException("Unsupported operator " + filter.getOperator());
+ }
+ }
+
+ private ConciseSet evalLogicalAnd(List<? extends TupleFilter> children) {
+ ConciseSet set = all();
+
+ for (TupleFilter c : children) {
+ ConciseSet t = evaluate(c);
+ if (t == null)
+ continue; // because it's AND
+
+ set.retainAll(t);
+ }
+ return set;
+ }
+
+ private ConciseSet evalLogicalOr(List<? extends TupleFilter> children) {
+ ConciseSet set = new ConciseSet();
+
+ for (TupleFilter c : children) {
+ ConciseSet t = evaluate(c);
+ if (t == null)
+ return null; // because it's OR
+
+ set.addAll(t);
+ }
+ return set;
+ }
+
+ private ConciseSet evalLogicalNot(List<? extends TupleFilter> children) {
+ return all();
+ }
+
+ private ConciseSet all() {
+ return not(new ConciseSet());
+ }
+
+ private ConciseSet not(ConciseSet set) {
+ set.add(indexedBlocks);
+ set.complement();
+ return set;
+ }
+
+ private int col(CompareTupleFilter filter) {
+ return filter.getColumn().getColumn().getZeroBasedIndex();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndexOfColumn.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndexOfColumn.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndexOfColumn.java
new file mode 100644
index 0000000..810a7a3
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndexOfColumn.java
@@ -0,0 +1,116 @@
+package org.apache.kylin.storage.gridtable;
+
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.util.NavigableMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
+
+import com.google.common.collect.Maps;
+
+public class GTInvertedIndexOfColumn {
+
+ final private IFilterCodeSystem<ByteArray> codeSystem;
+ final private ReentrantReadWriteLock rwLock;
+
+ private int nBlocks;
+ private NavigableMap<ByteArray, ConciseSet> rangeIndex;
+ private ConciseSet nullIndex;
+
+ public GTInvertedIndexOfColumn(IFilterCodeSystem<ByteArray> codeSystem) {
+ this.codeSystem = codeSystem;
+ this.rwLock = new ReentrantReadWriteLock();
+ this.rangeIndex = Maps.newTreeMap(codeSystem);
+ this.nullIndex = new ConciseSet();
+ }
+
+ public void add(Iterable<ByteArray> codes, int blockId) {
+ rwLock.writeLock().lock();
+ try {
+ for (ByteArray code : codes) {
+ if (codeSystem.isNull(code)) {
+ nullIndex.add(blockId);
+ continue;
+ }
+ ConciseSet set = rangeIndex.get(code);
+ if (set == null) {
+ set = new ConciseSet();
+ rangeIndex.put(code.copy(), set);
+ }
+ set.add(blockId);
+ }
+
+ if (blockId >= nBlocks) {
+ nBlocks = blockId + 1;
+ }
+
+ } finally {
+ rwLock.writeLock().unlock();
+ }
+ }
+
+ public ConciseSet getNull() {
+ rwLock.readLock().lock();
+ try {
+ return nullIndex.clone();
+ } finally {
+ rwLock.readLock().unlock();
+ }
+ }
+
+ public ConciseSet getEquals(ByteArray code) {
+ rwLock.readLock().lock();
+ try {
+ ConciseSet set = rangeIndex.get(code);
+ if (set == null)
+ return new ConciseSet();
+ else
+ return set.clone();
+ } finally {
+ rwLock.readLock().unlock();
+ }
+ }
+
+ public ConciseSet getIn(Iterable<ByteArray> codes) {
+ rwLock.readLock().lock();
+ try {
+ ConciseSet r = new ConciseSet();
+ for (ByteArray code : codes) {
+ ConciseSet set = rangeIndex.get(code);
+ if (set != null)
+ r.addAll(set);
+ }
+ return r;
+ } finally {
+ rwLock.readLock().unlock();
+ }
+ }
+
+ public ConciseSet getRange(ByteArray from, boolean fromInclusive, ByteArray to, boolean toInclusive) {
+ rwLock.readLock().lock();
+ try {
+ ConciseSet r = new ConciseSet();
+ if (from == null && to == null) {
+ r.add(nBlocks);
+ r.complement();
+ return r;
+ }
+ NavigableMap<ByteArray, ConciseSet> subMap;
+ if (from == null) {
+ subMap = rangeIndex.headMap(to, toInclusive);
+ } else if (to == null) {
+ subMap = rangeIndex.tailMap(from, fromInclusive);
+ } else {
+ subMap = rangeIndex.subMap(from, fromInclusive, to, toInclusive);
+ }
+ for (ConciseSet set : subMap.values()) {
+ r.addAll(set);
+ }
+ return r;
+ } finally {
+ rwLock.readLock().unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
index f45f126..978e3cf 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
@@ -36,27 +36,13 @@ class GTRawScanner implements IGTScanner {
ByteArray start = pkStart == null ? null : pkStart.exportColumns(info.primaryKey);
ByteArray endEx = pkEndExclusive == null ? null : pkEndExclusive.exportColumns(info.primaryKey);
- this.selectedColBlocks = computeHitColumnBlocks(columns);
+ this.selectedColBlocks = info.selectColumnBlocks(columns);
this.storeScanner = store.scan(start, endEx, selectedColBlocks, filterPushDown);
this.oneRecord = new GTRecord(info);
this.oneTuple = new TupleAdapter(oneRecord);
}
- private BitSet computeHitColumnBlocks(BitSet columns) {
- if (columns == null)
- columns = info.colAll;
-
- BitSet result = new BitSet();
- for (int i = 0; i < info.colBlocks.length; i++) {
- BitSet cb = info.colBlocks[i];
- if (cb.intersects(columns)) {
- result.set(i);
- }
- }
- return result;
- }
-
@Override
public int getScannedRowCount() {
return scannedRowCount;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 3f1ba91..2950fad 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -20,6 +20,14 @@ public class GTRecord implements Comparable<GTRecord> {
this.cols[i] = new ByteArray();
this.maskForEqualHashComp = info.colAll;
}
+
+ public ByteArray get(int i) {
+ return cols[i];
+ }
+
+ public void set(int i, ByteArray data) {
+ cols[i].set(data.array(), data.offset(), data.length());
+ }
/** set record to the codes of specified values, new space allocated to hold the codes */
public GTRecord setValues(Object... values) {
@@ -32,10 +40,6 @@ public class GTRecord implements Comparable<GTRecord> {
ByteBuffer buf = space.asBuffer();
int pos = buf.position();
for (int i = 0; i < info.nColumns; i++) {
- if (values[i] == null) {
- cols[i].set(null, 0, 0);
- continue;
- }
info.codeSystem.encodeColumnValue(i, values[i], buf);
int newPos = buf.position();
cols[i].set(buf.array(), buf.arrayOffset() + pos, newPos - pos);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlockIndex.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlockIndex.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlockIndex.java
deleted file mode 100644
index 21b0a9a..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlockIndex.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-public class GTRowBlockIndex {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
index edd70e4..b7f12df 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
@@ -38,7 +38,13 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
this.filterCS = new IFilterCodeSystem<ByteArray>() {
@Override
public boolean isNull(ByteArray code) {
- return (code == null || code.length() == 0);
+ // all 0xff is null
+ byte[] array = code.array();
+ for (int i = 0, j = code.offset(), n = code.length(); i < n; i++, j++) {
+ if (array[j] != (byte) 0xff)
+ return false;
+ }
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
new file mode 100644
index 0000000..b9f2a73
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
@@ -0,0 +1,23 @@
+package org.apache.kylin.storage.gridtable;
+
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class GTUtil {
+
+ static final TableDesc MOCKUP_TABLE = new TableDesc();
+ static {
+ MOCKUP_TABLE.setName("GT_MOCKUP_TABLE");
+ }
+
+ public static TblColRef tblColRef(int col, String datatype) {
+ ColumnDesc desc = new ColumnDesc();
+ String id = "" + (col + 1);
+ desc.setId(id);
+ desc.setName(id);
+ desc.setDatatype(datatype);
+ desc.init(MOCKUP_TABLE);
+ return new TblColRef(desc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java
new file mode 100644
index 0000000..1460039
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/GTInvertedIndexTest.java
@@ -0,0 +1,165 @@
+package org.apache.kylin.storage.gridtable;
+
+import static org.junit.Assert.*;
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.serializer.StringSerializer;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class GTInvertedIndexTest {
+
+ GTInfo info;
+ GTInvertedIndex index;
+ ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
+ ArrayList<ConciseSet> basicResults = Lists.newArrayList();
+
+ public GTInvertedIndexTest() {
+
+ info = GridTableTest.advancedInfo();
+ TblColRef colA = info.colRef(0);
+
+ // block i contains value "i", the last is NULL
+ index = new GTInvertedIndex(info);
+ GTRowBlock mockBlock = GTRowBlock.allocate(info);
+ GTRowBlock.Writer writer = mockBlock.getWriter();
+ GTRecord record = new GTRecord(info);
+ for (int i = 0; i < 10; i++) {
+ record.setValues(i < 9 ? "" + i : null, "", "", new LongWritable(0), new BigDecimal(0));
+ for (int j = 0; j < info.getRowBlockSize(); j++) {
+ writer.append(record);
+ }
+ writer.readyForFlush();
+ index.add(mockBlock);
+
+ writer.clearForNext();
+ }
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
+ basicResults.add(set(9));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
+ basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
+ basicResults.add(set(0));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
+ basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
+ basicResults.add(set(0, 5));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
+ basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
+ basicResults.add(set(0, 1, 2));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
+ basicResults.add(set(0, 1, 2, 3));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
+ basicResults.add(set(4, 5, 6, 7, 8));
+
+ basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
+ basicResults.add(set(3, 4, 5, 6, 7, 8));
+ }
+
+ @Test
+ public void testBasics() {
+ for (int i = 0; i < basicFilters.size(); i++) {
+ assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
+ }
+ }
+
+ @Test
+ public void testLogicalAnd() {
+ for (int i = 0; i < basicFilters.size(); i++) {
+ for (int j = 0; j < basicFilters.size(); j++) {
+ LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
+ ConciseSet r = basicResults.get(i).clone();
+ r.retainAll(basicResults.get(j));
+ assertEquals(r, index.filter(f));
+ }
+ }
+ }
+
+ @Test
+ public void testLogicalOr() {
+ for (int i = 0; i < basicFilters.size(); i++) {
+ for (int j = 0; j < basicFilters.size(); j++) {
+ LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
+ ConciseSet r = basicResults.get(i).clone();
+ r.addAll(basicResults.get(j));
+ assertEquals(r, index.filter(f));
+ }
+ }
+ }
+
+ @Test
+ public void testNotEvaluable() {
+ ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
+ assertEquals(all, index.filter(notEvaluable));
+
+ LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
+ assertEquals(all, index.filter(or));
+
+ LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
+ assertEquals(basicResults.get(0), index.filter(and));
+ }
+
+ public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
+ CompareTupleFilter filter = new CompareTupleFilter(op);
+ filter.addChild(columnFilter(col));
+ for (int i : ids) {
+ filter.addChild(constFilter(i));
+ }
+ return filter;
+ }
+
+ public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
+ LogicalTupleFilter filter = new LogicalTupleFilter(op);
+ for (TupleFilter f : filters)
+ filter.addChild(f);
+ return filter;
+ }
+
+ public static ColumnTupleFilter columnFilter(TblColRef col) {
+ return new ColumnTupleFilter(col);
+ }
+
+ public static ConstantTupleFilter constFilter(int id) {
+ byte[] space = new byte[10];
+ ByteBuffer buf = ByteBuffer.wrap(space);
+ StringSerializer stringSerializer = new StringSerializer();
+ stringSerializer.serialize("" + id, buf);
+ ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
+ return new ConstantTupleFilter(data);
+ }
+
+ public static ConciseSet set(int... ints) {
+ ConciseSet set = new ConciseSet();
+ for (int i : ints)
+ set.add(i);
+ return set;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05c2d096/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
index 899eb50..95f675f 100644
--- a/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/GridTableTest.java
@@ -168,13 +168,13 @@ public class GridTableTest {
System.out.println("Written Row Count: " + builder.getWrittenRowCount());
}
- private GTInfo basicInfo() {
+ public static GTInfo basicInfo() {
Builder builder = infoBuilder();
GTInfo info = builder.build();
return info;
}
- private GTInfo advancedInfo() {
+ public static GTInfo advancedInfo() {
Builder builder = infoBuilder();
builder.enableColumnBlock(new BitSet[] { setOf(0, 1, 2), setOf(3, 4) });
builder.enableRowBlock(4);
@@ -182,7 +182,7 @@ public class GridTableTest {
return info;
}
- private Builder infoBuilder() {
+ private static Builder infoBuilder() {
Builder builder = GTInfo.builder();
builder.setCodeSystem(new GTSampleCodeSystem());
builder.setColumns( //
@@ -193,10 +193,11 @@ public class GridTableTest {
DataType.getInstance("decimal") //
);
builder.setPrimaryKey(setOf(0));
+ builder.setColumnPreferIndex(setOf(0));
return builder;
}
- private BitSet setOf(int... values) {
+ private static BitSet setOf(int... values) {
BitSet set = new BitSet();
for (int i : values)
set.set(i);
[25/50] [abbrv] incubator-kylin git commit: KYLIN-650 remove
reference to wiki
Posted by li...@apache.org.
KYLIN-650 remove reference to wiki
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7faa216c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7faa216c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7faa216c
Branch: refs/heads/streaming-localdict
Commit: 7faa216c056d3076414ce5b350bf6c432b162bbb
Parents: f22582d
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 19:20:55 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 19:20:55 2015 -0700
----------------------------------------------------------------------
docs/Tutorial/Kylin and Tableau Tutorial.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7faa216c/docs/Tutorial/Kylin and Tableau Tutorial.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin and Tableau Tutorial.md b/docs/Tutorial/Kylin and Tableau Tutorial.md
index a2b28b2..dad90a8 100644
--- a/docs/Tutorial/Kylin and Tableau Tutorial.md
+++ b/docs/Tutorial/Kylin and Tableau Tutorial.md
@@ -25,7 +25,7 @@ Enter your Sever location and credentials: server host, port, username and passw
-Click "Connect" to get the list of projects that you have permission to access. See details about permission in [Kylin Cube Permission Grant Tutorial](https://github.com/KylinOLAP/Kylin/wiki/Kylin-Cube-Permission-Grant-Tutorial). Then choose the project you want to connect in the drop down list.
+Click "Connect" to get the list of projects that you have permission to access. See details about permission in [Kylin Cube Permission Grant Tutorial](Kylin Cube Permission Grant Tutorial.md). Then choose the project you want to connect in the drop down list.
[36/50] [abbrv] incubator-kylin git commit: KYLIN-650 typo
Posted by li...@apache.org.
KYLIN-650 typo
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c66a74b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c66a74b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c66a74b3
Branch: refs/heads/streaming-localdict
Commit: c66a74b37dc03f6115bc69c5de64b747d893719f
Parents: 269eac0
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 23:55:15 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 23:55:15 2015 -0700
----------------------------------------------------------------------
README.md | 2 +-
docs/MISC/Kylin Document Convensions.md | 9 ---------
docs/MISC/Kylin Document Conventions.md | 9 +++++++++
3 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c66a74b3/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 39bb3e1..144e01b 100644
--- a/README.md
+++ b/README.md
@@ -69,6 +69,6 @@ Resources
* Apache Proposal: [Apache Kylin](https://wiki.apache.org/incubator/KylinProposal)
-* [Kylin Document Convensions](docs/MISC/Kylin Document Convensions.md)
+* [Kylin Document Conventions](docs/MISC/Kylin Document Conventions.md)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c66a74b3/docs/MISC/Kylin Document Convensions.md
----------------------------------------------------------------------
diff --git a/docs/MISC/Kylin Document Convensions.md b/docs/MISC/Kylin Document Convensions.md
deleted file mode 100644
index a5b4fa0..0000000
--- a/docs/MISC/Kylin Document Convensions.md
+++ /dev/null
@@ -1,9 +0,0 @@
-Kylin Document Convensions
------
-
-1. Keep documentation in markdown files
-2. In markdown files, try to use relative URI for document/image reference.
-3. Never link to github wiki pages
-4. Try not to simply list all docuemnts in one page. Instead, try to organize different documents into a "story", so that readers are presented deeper materials as they proceed.
-5. Always keep documents updated
-6. ...
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c66a74b3/docs/MISC/Kylin Document Conventions.md
----------------------------------------------------------------------
diff --git a/docs/MISC/Kylin Document Conventions.md b/docs/MISC/Kylin Document Conventions.md
new file mode 100644
index 0000000..5e42852
--- /dev/null
+++ b/docs/MISC/Kylin Document Conventions.md
@@ -0,0 +1,9 @@
+Kylin Document Conventions
+-----
+
+1. Keep documentation in markdown files
+2. In markdown files, try to use relative URI for document/image reference.
+3. Never link to github wiki pages
+4. Try not to simply list all docuemnts in one page. Instead, try to organize different documents into a "story", so that readers are presented deeper materials as they proceed.
+5. Always keep documents updated
+6. ...
\ No newline at end of file
[38/50] [abbrv] incubator-kylin git commit: fix bug
Posted by li...@apache.org.
fix bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4a5c9ceb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4a5c9ceb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4a5c9ceb
Branch: refs/heads/streaming-localdict
Commit: 4a5c9ceb7cffcc461ef26b7a6b86caa9025fdbd3
Parents: 55c8257
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 20 15:21:49 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 20 15:21:49 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/metadata/filter/CompareTupleFilter.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4a5c9ceb/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 899320f..8ff3b64 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -79,7 +79,9 @@ public class CompareTupleFilter extends TupleFilter {
this.firstCondValue = this.conditionValues.iterator().next();
} else if (child instanceof DynamicTupleFilter) {
DynamicTupleFilter dynamicFilter = (DynamicTupleFilter) child;
- this.dynamicVariables.put(dynamicFilter.getVariableName(), null);
+ if (!this.dynamicVariables.containsKey(dynamicFilter.getVariableName())) {
+ this.dynamicVariables.put(dynamicFilter.getVariableName(), null);
+ }
}
}
[11/50] [abbrv] incubator-kylin git commit: add eagle.properties
Posted by li...@apache.org.
add eagle.properties
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1a46c2d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1a46c2d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1a46c2d1
Branch: refs/heads/streaming-localdict
Commit: 1a46c2d11fd3c67de9c055372f486a0316a29ffb
Parents: 1236500
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 19:42:34 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 19:42:34 2015 +0800
----------------------------------------------------------------------
.../kylin/streaming/Nous/NousEternalStreamProducer.java | 2 +-
.../java/org/apache/kylin/streaming/Nous/NousMessage.java | 2 +-
.../org/apache/kylin/streaming/Nous/NousMessageTest.java | 2 +-
.../test/resources/kafka_streaming_test/eagle.properties | 10 ++++++++++
4 files changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a46c2d1/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousEternalStreamProducer.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousEternalStreamProducer.java b/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousEternalStreamProducer.java
index c8bbc26..a93128f 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousEternalStreamProducer.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousEternalStreamProducer.java
@@ -1,4 +1,4 @@
-package org.apache.kylin.streaming.Nous;
+package org.apache.kylin.streaming.nous;
import java.util.Calendar;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a46c2d1/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessage.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessage.java b/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessage.java
index 91d3d75..3606514 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessage.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessage.java
@@ -1,4 +1,4 @@
-package org.apache.kylin.streaming.Nous;
+package org.apache.kylin.streaming.nous;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a46c2d1/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessageTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessageTest.java b/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessageTest.java
index b154c44..db0db81 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessageTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/Nous/NousMessageTest.java
@@ -1,4 +1,4 @@
-package org.apache.kylin.streaming.Nous;
+package org.apache.kylin.streaming.nous;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a46c2d1/streaming/src/test/resources/kafka_streaming_test/eagle.properties
----------------------------------------------------------------------
diff --git a/streaming/src/test/resources/kafka_streaming_test/eagle.properties b/streaming/src/test/resources/kafka_streaming_test/eagle.properties
new file mode 100644
index 0000000..02af8eb
--- /dev/null
+++ b/streaming/src/test/resources/kafka_streaming_test/eagle.properties
@@ -0,0 +1,10 @@
+zookeeper=druid-test-host1-556191.slc01.dev.ebayc3.com:2181
+
+brokers=druid-test-host1-556191.slc01.dev.ebayc3.com:9092,druid-test-host2-550753.slc01.dev.ebayc3.com:9092, druid-test-host3-550755.slc01.dev.ebayc3.com:9092
+
+consumer.timeout=60000
+consumer.bufferSize=65536
+consumer.maxReadCount=1000
+
+topic=cronus_system_metrics
+partitionId=0
[47/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d4d0675b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d4d0675b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d4d0675b
Branch: refs/heads/streaming-localdict
Commit: d4d0675b330e725c687688de1bcdd6b4aceb7f38
Parents: 4306645 a7b771d
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Mar 20 18:14:03 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Mar 20 18:14:03 2015 +0800
----------------------------------------------------------------------
README.md | 71 ++-----------
bin/kylin.sh | 18 ++--
bin/setenv.sh | 36 +++++++
...equently Asked Questions on Installation.pdf | Bin 52261 -> 0 bytes
docs/Installation/Hadoop Environment.md | 21 ++++
.../How to generate install binaries.md | 6 ++
docs/Installation/Installation.md | 38 +++++++
.../Installation/Multiple Kylin REST servers.md | 9 ++
...f Hadoop CLI Installation (Dev Env Setup).md | 76 -------------
... Hadoop CLI Installation (Dev Env Setup).pdf | Bin 44909 -> 0 bytes
.../Installation/Off Hadoop CLI Installation.md | 67 ++++++++++++
docs/Installation/On Hadoop CLI installation.md | 106 +++----------------
.../Installation/On Hadoop CLI installation.pdf | Bin 704398 -> 0 bytes
...On Hadoop Kylin installation using Docker.md | 2 +-
...n Hadoop Kylin installation using Docker.pdf | Bin 40098 -> 0 bytes
docs/Installation/off_cli_install_scene.png | Bin 0 -> 72567 bytes
docs/Installation/on_cli_install_scene.png | Bin 0 -> 53722 bytes
docs/README.md | 0
...in Cube Build and Job Monitoring Tutorial.md | 10 +-
...n Cube Build and Job Monitoring Tutorial.pdf | Bin 1054988 -> 0 bytes
docs/Tutorial/Kylin Cube Creation Tutorial.md | 9 +-
docs/Tutorial/Kylin Cube Creation Tutorial.pdf | Bin 1488946 -> 0 bytes
.../Kylin Cube Permission Grant Tutorial.pdf | Bin 358047 -> 0 bytes
docs/Tutorial/Kylin ODBC Driver Tutorial.md | 6 ++
docs/Tutorial/Kylin ODBC Driver Tutorial.pdf | Bin 34429 -> 0 bytes
docs/Tutorial/Kylin Web Tutorial.md | 8 +-
docs/Tutorial/Kylin Web Tutorial.pdf | Bin 1714951 -> 0 bytes
docs/Tutorial/Kylin and Excel Tutorial.md | 3 +
docs/Tutorial/Kylin and Tableau Tutorial.md | 4 +-
docs/Tutorial/Kylin and Tableau Tutorial.pdf | Bin 1246553 -> 0 bytes
docs/Tutorial/Quick play with a sample cube.md | 20 ++++
.../invertedindex/index/BatchSliceBuilder.java | 100 +++++++++++++++++
.../invertedindex/model/IIKeyValueCodec.java | 2 +-
.../apache/kylin/job/BuildIIWithStreamTest.java | 22 +++-
.../job/hadoop/cube/BaseCuboidJobTest.java | 3 +
.../job/hadoop/cube/MergeCuboidJobTest.java | 4 +-
.../kylin/job/hadoop/cube/NDCuboidJobTest.java | 4 +-
.../cube/RangeKeyDistributionJobTest.java | 4 +-
.../metadata/filter/CompareTupleFilter.java | 4 +-
old_README.md | 94 ++++++++++++++++
pom.xml | 4 +
query/src/test/resources/query/sql/query78.sql | 4 +
.../kylin/storage/hbase/CubeStorageEngine.java | 15 ++-
.../endpoint/EndpointTupleIterator.java | 2 +-
.../kylin/storage/filter/FilterBaseTest.java | 8 ++
.../storage/filter/FilterSerializeTest.java | 16 +++
.../.settings/org.eclipse.core.resources.prefs | 1 +
.../invertedindex/IIStreamBuilder.java | 37 ++++---
webapp/app/partials/query/query_detail.html | 2 +-
49 files changed, 557 insertions(+), 279 deletions(-)
----------------------------------------------------------------------
[09/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6fded261
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6fded261
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6fded261
Branch: refs/heads/streaming-localdict
Commit: 6fded26174d0f69cdb49a37cd04cebd9fdd7e5f7
Parents: 2695345
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 18:54:00 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 18:54:00 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/BuildIIWithStreamTest.java | 14 ++++++--------
.../coprocessor/endpoint/EndpointTupleIterator.java | 1 -
2 files changed, 6 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6fded261/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 19029cb..8c8d580 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -85,7 +85,7 @@ public class BuildIIWithStreamTest {
private static final Logger logger = LoggerFactory.getLogger(BuildIIWithStreamTest.class);
- private static final String[] II_NAME = new String[]{"test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
+ private static final String[] II_NAME = new String[]{"test_kylin_ii_left_join", "test_kylin_ii_inner_join"};
private IIManager iiManager;
private KylinConfig kylinConfig;
@@ -116,13 +116,6 @@ public class BuildIIWithStreamTest {
@After
public void after() throws Exception {
- for (String iiInstance : II_NAME) {
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii);
- }
- }
backup();
}
@@ -242,6 +235,11 @@ public class BuildIIWithStreamTest {
public void test() throws Exception {
for (String iiName : II_NAME) {
buildII(iiName);
+ IIInstance ii = iiManager.getII(iiName);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6fded261/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 805b490..cb543c7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -336,7 +336,6 @@ public class EndpointTupleIterator implements ITupleIterator {
}
final String columnName = columnNames.get(i);
if (tableRecordInfo.getDigest().isMetrics(i)) {
- logger.info("column name:" + columnName + " column value:" + tableRecord.getValueString(i));
tuple.setMeasureValue(columnName, tableRecord.getValueString(i));
} else {
tuple.setDimensionValue(columnName, columnValues.get(i));
[13/50] [abbrv] incubator-kylin git commit: KYLIN-625,
refactor aggregate scanner to decoration pattern
Posted by li...@apache.org.
KYLIN-625, refactor aggregate scanner to decoration pattern
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/73ad4d4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/73ad4d4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/73ad4d4b
Branch: refs/heads/streaming-localdict
Commit: 73ad4d4be50016c18cc80124199c2e97d3810a63
Parents: 5925320
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Mar 19 16:48:38 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Mar 19 16:48:38 2015 +0800
----------------------------------------------------------------------
.../storage/gridtable/GTAggregateScanner.java | 18 +++++++++---------
.../apache/kylin/storage/gridtable/GTInfo.java | 2 +-
.../kylin/storage/gridtable/GTRawScanner.java | 5 +++++
.../apache/kylin/storage/gridtable/GridTable.java | 12 ++++++++----
.../kylin/storage/gridtable/IGTScanner.java | 2 ++
5 files changed, 25 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
index 6cc82b4..ed38591 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
@@ -7,7 +7,6 @@ import java.util.Iterator;
import java.util.Map.Entry;
import java.util.SortedMap;
-import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import com.google.common.collect.Maps;
@@ -18,26 +17,27 @@ public class GTAggregateScanner implements IGTScanner {
final BitSet dimensions;
final BitSet metrics;
final String[] metricsAggrFuncs;
- final GTRawScanner rawScanner;
+ final IGTScanner rawScanner;
- GTAggregateScanner(GTInfo info, IGTStore store, GTRecord pkStart, GTRecord pkEndExclusive, BitSet dimensions, BitSet metrics, String[] metricsAggrFuncs, TupleFilter filterPushDown) {
+ GTAggregateScanner(IGTScanner rawScanner, BitSet dimensions, BitSet metrics, String[] metricsAggrFuncs) {
if (dimensions.intersects(metrics))
throw new IllegalStateException();
if (metrics.cardinality() != metricsAggrFuncs.length)
throw new IllegalStateException();
- this.info = info;
+ this.info = rawScanner.getInfo();
this.dimensions = dimensions;
this.metrics = metrics;
this.metricsAggrFuncs = metricsAggrFuncs;
-
- BitSet columns = new BitSet();
- columns.or(dimensions);
- columns.or(metrics);
- this.rawScanner = new GTRawScanner(info, store, pkStart, pkEndExclusive, columns, filterPushDown);
+ this.rawScanner = rawScanner;
}
@Override
+ public GTInfo getInfo() {
+ return info;
+ }
+
+ @Override
public int getScannedRowCount() {
return rawScanner.getScannedRowCount();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
index 97cac87..04e720c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -25,7 +25,7 @@ public class GTInfo {
transient TblColRef[] colRefs;
// grid info
- BitSet primaryKey; // columns sorted and unique
+ BitSet primaryKey; // order by, uniqueness is not required
BitSet[] colBlocks; // at least one column block
BitSet colBlocksAll;
int rowBlockSize; // 0: disable row block
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
index 978e3cf..737cd41 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
@@ -44,6 +44,11 @@ class GTRawScanner implements IGTScanner {
}
@Override
+ public GTInfo getInfo() {
+ return info;
+ }
+
+ @Override
public int getScannedRowCount() {
return scannedRowCount;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
index d0304bc..2b49617 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
@@ -24,12 +24,12 @@ public class GridTable {
assert shard < info.nShards;
return new GTBuilder(info, shard, store);
}
-
+
public GTBuilder append() {
assert info.isShardingEnabled() == false;
return append(-1);
}
-
+
public GTBuilder append(int shard) {
return new GTBuilder(info, shard, store, true);
}
@@ -40,13 +40,17 @@ public class GridTable {
public IGTScanner scanAndAggregate(GTRecord pkStart, GTRecord pkEndExclusive, BitSet dimensions, //
BitSet metrics, String[] metricsAggrFuncs, TupleFilter filterPushDown) throws IOException {
- return new GTAggregateScanner(info, store, pkStart, pkEndExclusive, dimensions, metrics, metricsAggrFuncs, filterPushDown);
+ BitSet columns = new BitSet();
+ columns.or(dimensions);
+ columns.or(metrics);
+ IGTScanner rawScanner = scan(pkStart, pkEndExclusive, columns, filterPushDown);
+ return new GTAggregateScanner(rawScanner, dimensions, metrics, metricsAggrFuncs);
}
public GTInfo getInfo() {
return info;
}
-
+
public IGTStore getStore() {
return store;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
index 8a8c79b..285a301 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
@@ -4,6 +4,8 @@ import java.io.Closeable;
public interface IGTScanner extends Iterable<GTRecord>, Closeable {
+ public GTInfo getInfo();
+
public int getScannedRowCount();
public int getScannedRowBlockCount();
[06/50] [abbrv] incubator-kylin git commit: fix ci
Posted by li...@apache.org.
fix ci
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/26953451
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/26953451
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/26953451
Branch: refs/heads/streaming-localdict
Commit: 2695345179129382d160ac32f30988a88039ada8
Parents: 2fce2ba
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 17:42:54 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 17:42:54 2015 +0800
----------------------------------------------------------------------
.../kylin/invertedindex/index/TableRecordInfoDigest.java | 11 +++++++++--
.../coprocessor/endpoint/EndpointTupleIterator.java | 9 ++++++++-
2 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/26953451/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
index 12f88bf..832a39d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
@@ -41,6 +41,7 @@ public class TableRecordInfoDigest {
private int[] dictMaxIds;// max id for each of the dict
private int[] lengths;// length of each encoded dict
private boolean[] isMetric;// whether it's metric or dict
+ private FixedLenMeasureCodec[] measureCodecs;
public TableRecordInfoDigest(int nColumns, int byteFormLen, int[] offsets,
int[] dictMaxIds, int[] lengths, boolean[] isMetric,
@@ -52,7 +53,13 @@ public class TableRecordInfoDigest {
this.lengths = lengths;
this.isMetric = isMetric;
this.metricDataTypes = metricDataTypes;
- }
+ this.measureCodecs = new FixedLenMeasureCodec[nColumns];
+ for (int i = 0; i < isMetric.length; i++) {
+ if (isMetric[i]) {
+ measureCodecs[i] = FixedLenMeasureCodec.get(DataType.getInstance(metricDataTypes[i]));
+ }
+ }
+ }
private TableRecordInfoDigest() {
}
@@ -120,7 +127,7 @@ public class TableRecordInfoDigest {
@SuppressWarnings("unchecked")
public FixedLenMeasureCodec<LongWritable> codec(int col) {
// yes, all metrics are long currently
- return (FixedLenMeasureCodec<LongWritable>) FixedLenMeasureCodec.get(DataType.getInstance(metricDataTypes[col]));
+ return measureCodecs[col];
}
public static byte[] serialize(TableRecordInfoDigest o) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/26953451/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 31631cb..805b490 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
+import org.apache.kylin.metadata.measure.fixedlen.FixedPointLongCodec;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
@@ -333,7 +334,13 @@ public class EndpointTupleIterator implements ITupleIterator {
if (!tuple.hasColumn(column)) {
continue;
}
- tuple.setDimensionValue(columnNames.get(i), columnValues.get(i));
+ final String columnName = columnNames.get(i);
+ if (tableRecordInfo.getDigest().isMetrics(i)) {
+ logger.info("column name:" + columnName + " column value:" + tableRecord.getValueString(i));
+ tuple.setMeasureValue(columnName, tableRecord.getValueString(i));
+ } else {
+ tuple.setDimensionValue(columnName, columnValues.get(i));
+ }
}
if (measureValues != null) {
[18/50] [abbrv] incubator-kylin git commit: KYLIN-648
Posted by li...@apache.org.
KYLIN-648
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/014ef833
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/014ef833
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/014ef833
Branch: refs/heads/streaming-localdict
Commit: 014ef833b8f0eff75ca5240d23adf1973419bd0f
Parents: cf99527
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Mar 19 22:30:19 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Mar 19 22:30:19 2015 +0800
----------------------------------------------------------------------
query/src/test/resources/query/sql/query78.sql | 4 ++++
.../kylin/storage/hbase/CubeStorageEngine.java | 15 ++++++++++-----
2 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/014ef833/query/src/test/resources/query/sql/query78.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query78.sql b/query/src/test/resources/query/sql/query78.sql
new file mode 100644
index 0000000..f1d8fb2
--- /dev/null
+++ b/query/src/test/resources/query/sql/query78.sql
@@ -0,0 +1,4 @@
+select sum(PRICE), LSTG_FORMAT_NAME
+from test_kylin_fact
+where (LSTG_FORMAT_NAME in ('ABIN')) or (LSTG_FORMAT_NAME>='FP-GTC' and LSTG_FORMAT_NAME<='Others')
+group by LSTG_FORMAT_NAME
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/014ef833/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index a4ecc2a..8737e8c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -109,7 +109,7 @@ public class CubeStorageEngine implements IStorageEngine {
context.setExactAggregation(isExactAggregation);
// translate filter for scan range and compose returning groups for coprocessor, note:
- // - columns on evaluate-able filter have to return
+ // - columns on non-evaluatable filter have to return
// - columns on loosened filter (due to derived translation) have to return
Set<TblColRef> groupsCopD = Sets.newHashSet(groupsD);
groupsCopD.addAll(context.getOtherMandatoryColumns()); // TODO: this is tricky, to generalize
@@ -538,8 +538,10 @@ public class CubeStorageEngine implements IStorageEngine {
List<Pair<byte[], byte[]>> newFuzzyKeys = new ArrayList<Pair<byte[], byte[]>>(mergeSize);
List<Collection<ColumnValueRange>> newFlatOrAndFilter = Lists.newLinkedList();
+ boolean hasNonFuzzyRange = false;
for (int k = from; k <= to; k++) {
HBaseKeyRange nextRange = keyRanges.get(k);
+ hasNonFuzzyRange = hasNonFuzzyRange || nextRange.getFuzzyKeys().isEmpty();
newFuzzyKeys.addAll(nextRange.getFuzzyKeys());
newFlatOrAndFilter.addAll(nextRange.getFlatOrAndFilter());
if (Bytes.compareTo(stopKey, nextRange.getStopKey()) < 0) {
@@ -553,11 +555,14 @@ public class CubeStorageEngine implements IStorageEngine {
}
}
+ // if any range is non-fuzzy, then all fuzzy keys must be cleared
+ if (hasNonFuzzyRange) {
+ newFuzzyKeys.clear();
+ }
+
partitionColumnStartDate = (partitionColumnStartDate == Long.MAX_VALUE) ? 0 : partitionColumnStartDate;
partitionColumnEndDate = (partitionColumnEndDate == 0) ? Long.MAX_VALUE : partitionColumnEndDate;
- keyRange =
-
- new HBaseKeyRange(cubeSegment, cuboid, startKey, stopKey, newFuzzyKeys, newFlatOrAndFilter, partitionColumnStartDate, partitionColumnEndDate);
+ keyRange = new HBaseKeyRange(cubeSegment, cuboid, startKey, stopKey, newFuzzyKeys, newFlatOrAndFilter, partitionColumnStartDate, partitionColumnEndDate);
}
return keyRange;
}
@@ -605,7 +610,7 @@ public class CubeStorageEngine implements IStorageEngine {
}
private void setLimit(TupleFilter filter, StorageContext context) {
- boolean goodAggr = context.isExactAggregation() ;
+ boolean goodAggr = context.isExactAggregation();
boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled());
boolean goodSort = context.hasSort() == false;
if (goodAggr && goodFilter && goodSort) {
[20/50] [abbrv] incubator-kylin git commit: KYLIN-650 moving wikis to
code repository
Posted by li...@apache.org.
KYLIN-650 moving wikis to code repository
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/592e96b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/592e96b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/592e96b5
Branch: refs/heads/streaming-localdict
Commit: 592e96b519171ae9906cd1774b2a3e88e9dc3d91
Parents: a540f40
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 18:56:04 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 18:56:21 2015 -0700
----------------------------------------------------------------------
README.md | 71 ++-----------
...equently Asked Questions on Installation.pdf | Bin 52261 -> 0 bytes
docs/Installation/Hadoop Environment.md | 21 ++++
.../How to generate install binaries.md | 6 ++
docs/Installation/Installation.md | 38 +++++++
.../Installation/Multiple Kylin REST servers.md | 9 ++
...f Hadoop CLI Installation (Dev Env Setup).md | 76 -------------
... Hadoop CLI Installation (Dev Env Setup).pdf | Bin 44909 -> 0 bytes
.../Installation/Off Hadoop CLI Installation.md | 67 ++++++++++++
docs/Installation/On Hadoop CLI installation.md | 106 +++----------------
.../Installation/On Hadoop CLI installation.pdf | Bin 704398 -> 0 bytes
...On Hadoop Kylin installation using Docker.md | 2 +-
...n Hadoop Kylin installation using Docker.pdf | Bin 40098 -> 0 bytes
docs/Installation/install_and_run.md | 83 ---------------
docs/Installation/off_cli_install_scene.png | Bin 0 -> 72567 bytes
docs/Installation/on_cli_install_scene.png | Bin 0 -> 53722 bytes
docs/README.md | 0
...n Cube Build and Job Monitoring Tutorial.pdf | Bin 1054988 -> 0 bytes
docs/Tutorial/Kylin Cube Creation Tutorial.md | 9 +-
docs/Tutorial/Kylin Cube Creation Tutorial.pdf | Bin 1488946 -> 0 bytes
.../Kylin Cube Permission Grant Tutorial.pdf | Bin 358047 -> 0 bytes
docs/Tutorial/Kylin ODBC Driver Tutorial.md | 7 ++
docs/Tutorial/Kylin ODBC Driver Tutorial.pdf | Bin 34429 -> 0 bytes
docs/Tutorial/Kylin Web Tutorial.md | 8 +-
docs/Tutorial/Kylin Web Tutorial.pdf | Bin 1714951 -> 0 bytes
docs/Tutorial/Kylin and Excel Tutorial.md | 3 +
docs/Tutorial/Kylin and Tableau Tutorial.pdf | Bin 1246553 -> 0 bytes
docs/Tutorial/Quick play with a sample cube.md | 10 ++
old_README.md | 94 ++++++++++++++++
29 files changed, 294 insertions(+), 316 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 44f6859..28c2df6 100644
--- a/README.md
+++ b/README.md
@@ -2,83 +2,24 @@ Kylin OLAP
===========
-Overview
+What is Kylin
------------
> Extreme OLAP Engine for Big Data
An open source distributed Analytical Engine from eBay to provide SQL interface and multi-dimensional analysis (OLAP) on Hadoop to support TB to PB size analysis.
+This [slice](docs/Apache Kylin 2014 Dec.pdf) might help you better understand what kylin is and how it works.
-Getting Started
----------------
-### Prerequisites ###
-
-#### Hadoop ####
-- Hadoop: 2.2.0.2.0.6.0-61 or above
-- Hive: 0.12.0.2.0.6.0-61 or above
-- HBase: 0.96.0.2.0.6.0-61-hadoop2
-
-
-_Tested with Hortonworks HDP 2.1.3 and Cloudera Quickstart VM 5.1._
-
-#### Misc ####
-- Command hadoop, hive, hbase is workable on your hadoop cluster
-- JDK Runtime: JDK7 (OpenJDK or Oracle JDK)
-- Maven
-- Git
-- Tomcat (CATALINA_HOME being set)
-- Npm
-
-#### Before Install ####
-- For the quick tutorial, we assume that your hadoop has priviledges disabled which means any user could run hadoop and hive command
-
-
-Installation
+Tutorial
------------
-It is very easy to install Kylin for exploration/development. There are 3 supported options:
-
-
-1. Sandbox (HDP or CDH sandbox)
-2. Dev Environment (IDE + Sandbox)
-3. Docker container (Docker)
-
-#### Sandbox (On-Hadoop-CLI installation) ####
-
-If you are free to install Kylin on your hadoop CLI machine or Hadoop sandbox, this is the most convenient scenario, for it puts everything in a single machine.
-
-
-
+Please follow this installation tutorial to start with Kylin: [Installation Tutorial](docs/Installation/Installation.md)
-For normal users, the application in the above picture means `Kylin Web`, which contains a web interface for cube building, querying and all sorts of management. Kylin Web launches a query engine for querying and a cube build engine for building cubes. These two engines interact with the components in Hadoop CLI, like hive and hbase.
-For a hands-on tutorial please visit [On-Hadoop-CLI installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-CLI-installation).
-#### Dev Environment (Off-Hadoop-CLI Installation) ####
-
-This is typically for development environment setup.
-
-Applications does not necessarily mean Kylin Web, it can also be any test cases or java applications running at your local IDE(query engine and cube build engine can be launched programmatically in your code). In this case, it is no longer reasonable to assume that all the Kylin components reside in the same machine as your Hadoop CLI. Fortunately, Kylin still works under such condition with proper settings on your CLI machine.
-
-
-
-For a hands-on tutorial please visit [Off-Hadoop-CLI installation](https://github.com/KylinOLAP/Kylin/wiki/Off-Hadoop-CLI-Installation-(Dev-Env-Setup))
-
-#### Docker Container ####
-With help from [SequenceIQ](http://sequenceiq.com/), there's docker container for Kylin (along with Hadoop, HBase and Hive) available now:[sequenceiq/docker-kylin](https://github.com/sequenceiq/docker-kylin). The only thing you will need to do is to pull the container from the official Docker repository to be up and running in few minutes.
-
-Features:
-
-- Size - Light weight compared to downloading and setting up HDP or CDH sandbox.
-- Cluster support - Supports multi node installation.
-- Fully Automated - No manual steps. One command does it all
-
-For a hands-on tutorial please visit [Kylin Docker installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-Kylin-installation-using-Docker).
-
-
-
-### Resources ###
+Resources
+------------
* Web Site: <http://kylin.io>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/Frequently Asked Questions on Installation.pdf
----------------------------------------------------------------------
diff --git a/docs/Installation/Frequently Asked Questions on Installation.pdf b/docs/Installation/Frequently Asked Questions on Installation.pdf
deleted file mode 100644
index ff340ab..0000000
Binary files a/docs/Installation/Frequently Asked Questions on Installation.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/Hadoop Environment.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Hadoop Environment.md b/docs/Installation/Hadoop Environment.md
new file mode 100644
index 0000000..2e72527
--- /dev/null
+++ b/docs/Installation/Hadoop Environment.md
@@ -0,0 +1,21 @@
+## Hadoop Environment
+
+Kylin requires you having access to a hadoop CLI, where you have full permissions to hdfs, hive, hbase and map-reduce. To make things easier we strongly recommend you starting with running Kylin on a hadoop sandbox, like <http://hortonworks.com/products/hortonworks-sandbox/>. In the following tutorial we'll go with **Hortonworks Sandbox 2.1** and **Cloudera QuickStart VM 5.1**.
+
+To avoid permission issue, we suggest you using `root` account. The password for **Hortonworks Sandbox 2.1** is `hadoop` , for **Cloudera QuickStart VM 5.1** is `cloudera`.
+
+We also suggest you using bridged mode instead of NAT mode in your virtual box settings. Bridged mode will assign your sandbox an independent IP so that you can avoid issues like https://github.com/KylinOLAP/Kylin/issues/12
+
+### Start Hadoop
+
+Please make sure Hive, HDFS and HBase are available on our CLI machine.
+If you don't know how, here's a simple tutorial for hortonworks sanbox:
+
+Use ambari helps to launch hadoop:
+
+ ambari-agent start
+ ambari-server start
+
+With both command successfully run you can go to ambari homepage at <http://your_sandbox_ip:8080> (user:admin,password:admin) to check everything's status. **By default hortonworks ambari disables Hbase, you'll need manually start the `Hbase` service at ambari homepage.**
+
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/How to generate install binaries.md
----------------------------------------------------------------------
diff --git a/docs/Installation/How to generate install binaries.md b/docs/Installation/How to generate install binaries.md
new file mode 100644
index 0000000..d7946e0
--- /dev/null
+++ b/docs/Installation/How to generate install binaries.md
@@ -0,0 +1,6 @@
+###Build from source
+> git clone https://github.com/KylinOLAP/Kylin.git
+> cd KylinOLAP/Kylin
+> sh script/package.sh
+
+In order to generate binary package, **maven** and **npm** are pre-requisites.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/Installation.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Installation.md b/docs/Installation/Installation.md
new file mode 100644
index 0000000..5b33a56
--- /dev/null
+++ b/docs/Installation/Installation.md
@@ -0,0 +1,38 @@
+Installation
+------------
+
+### Prerequisites on hadoop###
+
+- Hadoop: 2.2.0.2.0.6.0-61 or above
+- Hive: 0.12.0.2.0.6.0-61 or above
+- HBase: 0.96.0.2.0.6.0-61-hadoop2
+
+_Tested with Hortonworks HDP 2.1.3 and Cloudera Quickstart VM 5.1._
+
+
+It is very easy to install Kylin for exploration/development. There are 3 installation scenarios:
+
+#### On-Hadoop-CLI installation ####
+
+If you are free to install Kylin on your hadoop CLI machine or Hadoop sandbox, this is the most convenient scenario, for it puts everything in a single machine.
+
+For a hands-on tutorial please visit [On-Hadoop-CLI installation](On Hadoop CLI installation.md).
+
+#### Dev Environment (Off-Hadoop-CLI Installation) ####
+
+This is typically for development environment setup.
+
+For a hands-on tutorial please visit [Off Hadoop CLI Installation (Dev Env Setup)](Off Hadoop CLI Installation.md).
+#### Docker Container ####
+With help from [SequenceIQ](http://sequenceiq.com/), there's docker container for Kylin (along with Hadoop, HBase and Hive) available now:[sequenceiq/docker-kylin](https://github.com/sequenceiq/docker-kylin). The only thing you will need to do is to pull the container from the official Docker repository to be up and running in few minutes.
+
+Features:
+
+- Size - Light weight compared to downloading and setting up HDP or CDH sandbox.
+- Cluster support - Supports multi node installation.
+- Fully Automated - No manual steps. One command does it all
+
+For a hands-on tutorial please visit [Kylin Docker installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-Kylin-installation-using-Docker).
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/Multiple Kylin REST servers.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Multiple Kylin REST servers.md b/docs/Installation/Multiple Kylin REST servers.md
new file mode 100644
index 0000000..2ee6832
--- /dev/null
+++ b/docs/Installation/Multiple Kylin REST servers.md
@@ -0,0 +1,9 @@
+### Multiple Kylin REST servers
+
+If you are running Kylin in a cluster or you have multiple Kylin REST server instances, please make sure you have the following property correctly configured in ${KYLIN_HOME}/conf/kylin.properties
+
+1. kylin.rest.servers
+ List of web servers in use, this enables one web server instance to sync up with other servers.
+
+2. kylin.server.mode
+ Make sure there is only one instance whose "kylin.server.mode" is set to "all" if there are multiple instances.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/Off Hadoop CLI Installation (Dev Env Setup).md
----------------------------------------------------------------------
diff --git a/docs/Installation/Off Hadoop CLI Installation (Dev Env Setup).md b/docs/Installation/Off Hadoop CLI Installation (Dev Env Setup).md
deleted file mode 100644
index ef7b205..0000000
--- a/docs/Installation/Off Hadoop CLI Installation (Dev Env Setup).md
+++ /dev/null
@@ -1,76 +0,0 @@
-Off Hadoop CLI Installation (Dev Env Setup)
-===
-Off-Hadoop-CLI installation is usually for **development use**.
-
-Developers want to run kylin test cases or applications at their development machine. The scenario is depicted at https://github.com/KylinOLAP/Kylin#off-hadoop-cli-installation.
-
-
-By following this tutorial, you will be able to build kylin test cubes by running a specific test case, and you can further run other test cases against the cubes having been built.
-
-
-## Environment on the Hadoop CLI
-
-Off-Hadoop-CLI installation requires you having a hadoop CLI machine (or a hadoop sandbox) as well as your local develop machine. To make things easier we strongly recommend you starting with running Kylin on a hadoop sandbox, like <http://hortonworks.com/products/hortonworks-sandbox/>. In the following tutorial we'll go with **Hortonworks Sandbox 2.1**.
-
-### Start Hadoop
-
-In Hortonworks sandbox, ambari helps to launch hadoop:
-
- ambari-agent start
- ambari-server start
-
-With both command successfully run you can go to ambari home page at <http://yoursandboxip:8080> (user:admin,password:admin) to check everything's status. By default ambari disables Hbase, you'll need manually start the `Hbase` service.
-
-For other hadoop distribution, basically start the hadoop cluster, make sure HDFS, YARN, Hive, HBase are running.
-
-
-## Environment on the dev machine
-
-### Install maven
-
-The latest maven can be found at <http://maven.apache.org/download.cgi>, we create a symbolic so that `mvn` can be run anywhere.
-
- cd ~
- wget http://apache.proserve.nl/maven/maven-3/3.2.3/binaries/apache-maven-3.2.3-bin.tar.gz
- tar -xzvf apache-maven-3.2.3-bin.tar.gz
- ln -s /root/apache-maven-3.2.3/bin/mvn /usr/bin/mvn
-
-### Compile
-
-First clone the Kylin project to your local:
-
- git clone https://github.com/KylinOLAP/Kylin.git
-
-Install Kylin artifacts to the maven repo
-
- mvn clean install -DskipTests
-
-### Modify local configuration
-
-Local configuration must be modified to point to your hadoop sandbox (or CLI) machine. If you are using a Hortonworks sandbox, this section may be skipped.
-
-* In **examples/test_case_data/sandbox/kylin.properties**
- * Find `sandbox` and replace with your hadoop hosts
- * Find `kylin.job.remote.cli.username` and `kylin.job.remote.cli.password`, fill in the user name and password used to login hadoop cluster for hadoop command execution
-
-* In **examples/test_case_data/sandbox**
- * For each configuration xml file, find all occurrence of `sandbox` and replace with your hadoop hosts
-
-An alternative to the host replacement is updating your `hosts` file to resolve `sandbox` and `sandbox.hortonworks.com` to the IP of your sandbox machine.
-
-### Run unit tests
-
-Run a end-to-end cube building test
-
- mvn test -Dtest=org.apache.kylin.job.BuildCubeWithEngineTest -DfailIfNoTests=false
-
-Run other tests, the end-to-end cube building test is exclueded
-
- mvn test
-
-### Launch Kylin Web Server
-
-In your Eclipse IDE, launch `org.apache.kylin.rest.DebugTomcat` with specifying VM arguments "-Dspring.profiles.active=sandbox". (By default Kylin server will listen on 7070 port; If you want to use another port, please specify it as a parameter when run `DebugTomcat)
-
-Check Kylin Web available at http://localhost:7070 (user:ADMIN,password:KYLIN)
-
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/Off Hadoop CLI Installation (Dev Env Setup).pdf
----------------------------------------------------------------------
diff --git a/docs/Installation/Off Hadoop CLI Installation (Dev Env Setup).pdf b/docs/Installation/Off Hadoop CLI Installation (Dev Env Setup).pdf
deleted file mode 100644
index 1cdf8da..0000000
Binary files a/docs/Installation/Off Hadoop CLI Installation (Dev Env Setup).pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/Off Hadoop CLI Installation.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Off Hadoop CLI Installation.md b/docs/Installation/Off Hadoop CLI Installation.md
new file mode 100644
index 0000000..d63bf13
--- /dev/null
+++ b/docs/Installation/Off Hadoop CLI Installation.md
@@ -0,0 +1,67 @@
+Off Hadoop CLI Installation (Dev Env Setup)
+===
+Off-Hadoop-CLI installation is usually for **development use**.
+
+Developers want to run kylin test cases or applications at their development machine. The scenario is depicted as:
+
+
+
+The "Applications" here can be any unit cases running on your dev machine's IDE. By following this tutorial, you will be able to build kylin test cubes by running a specific test case, and you can further run other test cases against the cubes having been built.
+
+
+## Environment on the Hadoop CLI
+
+### Environment
+
+Kylin Rquires a properly setup hadoop environment to run. Please take a look [this](Hadoop Environment.md).
+
+## Environment on the dev machine
+
+### Install maven
+
+The latest maven can be found at <http://maven.apache.org/download.cgi>, we create a symbolic so that `mvn` can be run anywhere.
+
+ cd ~
+ wget http://apache.proserve.nl/maven/maven-3/3.2.3/binaries/apache-maven-3.2.3-bin.tar.gz
+ tar -xzvf apache-maven-3.2.3-bin.tar.gz
+ ln -s /root/apache-maven-3.2.3/bin/mvn /usr/bin/mvn
+
+### Compile
+
+First clone the Kylin project to your local:
+
+ git clone https://github.com/KylinOLAP/Kylin.git
+
+Install Kylin artifacts to the maven repo
+
+ mvn clean install -DskipTests
+
+### Modify local configuration
+
+Local configuration must be modified to point to your hadoop sandbox (or CLI) machine. If you are using a Hortonworks sandbox, this section may be skipped.
+
+* In **examples/test_case_data/sandbox/kylin.properties**
+ * Find `sandbox` and replace with your hadoop hosts
+ * Find `kylin.job.remote.cli.username` and `kylin.job.remote.cli.password`, fill in the user name and password used to login hadoop cluster for hadoop command execution
+
+* In **examples/test_case_data/sandbox**
+ * For each configuration xml file, find all occurrence of `sandbox` and replace with your hadoop hosts
+
+An alternative to the host replacement is updating your `hosts` file to resolve `sandbox` and `sandbox.hortonworks.com` to the IP of your sandbox machine.
+
+### Run unit tests
+
+Run a end-to-end cube building test
+
+ mvn test -Dtest=org.apache.kylin.job.BuildCubeWithEngineTest -DfailIfNoTests=false
+
+Run other tests, the end-to-end cube building test is exclueded
+
+ mvn test
+
+### Launch Kylin Web Server
+
+In your Eclipse IDE, launch `org.apache.kylin.rest.DebugTomcat` with specifying VM arguments "-Dspring.profiles.active=sandbox". (By default Kylin server will listen on 7070 port; If you want to use another port, please specify it as a parameter when run `DebugTomcat)
+
+Check Kylin Web available at http://localhost:7070 (user:ADMIN,password:KYLIN)
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/On Hadoop CLI installation.md
----------------------------------------------------------------------
diff --git a/docs/Installation/On Hadoop CLI installation.md b/docs/Installation/On Hadoop CLI installation.md
index a718722..db2409b 100644
--- a/docs/Installation/On Hadoop CLI installation.md
+++ b/docs/Installation/On Hadoop CLI installation.md
@@ -1,102 +1,30 @@
On Hadoop CLI installation
===
-On-Hadoop-CLI installation is for demo use, or for those who want to host their own web site to provide Kylin service. The scenario is depicted at https://github.com/KylinOLAP/Kylin#on-hadoop-cli-installation.
+On-Hadoop-CLI installation is the most common way of installing Kylin. It can be used for demo use, or for those who want to host their own web site to provide Kylin service. The scenario is depicted as:
-Except for some prerequisite software installations, the core of Kylin installation is accomplished by running a single script. After running the script, you will be able to build sample cube and query the tables behind the cubes via a unified web interface.
-
-## Environment
-
-Running On-Hadoop-CLI installation requires you having access to a hadoop CLI, where you have full permissions to hdfs, hive, hbase and map-reduce. To make things easier we strongly recommend you starting with running Kylin on a hadoop sandbox, like <http://hortonworks.com/products/hortonworks-sandbox/>. In the following tutorial we'll go with **Hortonworks Sandbox 2.1** and **Cloudera QuickStart VM 5.1**.
-
-To avoid permission issue, we suggest you using `root` account. The password for **Hortonworks Sandbox 2.1** is `hadoop` , for **Cloudera QuickStart VM 5.1** is `cloudera`.
-
-We also suggest you using bridged mode instead of NAT mode in your virtual box settings. Bridged mode will assign your sandbox an independent IP so that you can avoid issues like https://github.com/KylinOLAP/Kylin/issues/12
-
-### Start Hadoop
-
-For Hortonworks, ambari helps to launch hadoop:
-
- ambari-agent start
- ambari-server start
-
-With both command successfully run you can go to ambari homepage at <http://your_sandbox_ip:8080> (user:admin,password:admin) to check everything's status. **By default hortonworks ambari disables Hbase, you'll need manually start the `Hbase` service at ambari homepage.**
-
-
-
-For Cloudera, you can skip this step since they're by default activated.
-
-### Install tomcat
-
-The latest tomcat can be found at <http://tomcat.apache.org/download-70.cgi>, we need the variable `CATALINA_HOME` exported.
+
- cd ~
- wget http://apache.cs.uu.nl/dist/tomcat/tomcat-7/v7.0.57/bin/apache-tomcat-7.0.57.tar.gz
- tar -xzvf apache-tomcat-7.0.57.tar.gz
- export CATALINA_HOME=~/apache-tomcat-7.0.57
+For normal use cases, the application in the above picture means Kylin Web, which contains a web interface for cube building, querying and all sorts of management. Kylin Web launches a query engine for querying and a cube build engine for building cubes. These two engines interact with the components in Hadoop CLI, like hive and hbase.
+Except for some prerequisite software installations, the core of Kylin installation is accomplished by running a single script. After running the script, you will be able to build sample cube and query the tables behind the cubes via a unified web interface.
-### Install maven:
-
-For Cloudera, you can skip this step since maven is by default installed.
-
-The latest maven can be found at <http://maven.apache.org/download.cgi>, we create a symbolic so that `mvn` can be run anywhere.
-
- cd ~
- wget http://apache.proserve.nl/maven/maven-3/3.2.3/binaries/apache-maven-3.2.3-bin.tar.gz
- tar -xzvf apache-maven-3.2.3-bin.tar.gz
- ln -s ~/apache-maven-3.2.3/bin/mvn /usr/bin/mvn
-
-
-### Install npm
-
-Npm comes with latest versions of nodejs, just donwload nodejs from <http://nodejs.org/download/>, `npm` is located at the `/bin` folder of `nodejs`, we append the folder to `PATH` so that `npm` can be run anywhere.
-
- cd ~
- wget http://nodejs.org/dist/v0.10.32/node-v0.10.32-linux-x64.tar.gz
- tar -xzvf node-v0.10.32-linux-x64.tar.gz
- export PATH=~/node-v0.10.32-linux-x64/bin:$PATH
-
-
-## Build cubes & Query tables
-
-First clone the Kylin project to your local:
-
- git clone https://github.com/KylinOLAP/Kylin.git
-
-Go into the folder and run deploy script:
-
- cd Kylin/
- ./deploy.sh
-
-If you meet any problems, please check [FAQ](https://github.com/KylinOLAP/Kylin/wiki/Frequently-Asked-Questions-on-Installation) first.
-This script will help to:
-
-1. Check your environment
-2. Build Kylin artifacts with Maven
-3. Generate some sample tables
-3. Create empty cubes for these tables
-4. Lauch a one-stop website to build cubes & query tables
+### Environment
-After successfully running the script, please navigate to <http://your_sandbox_ip:7070> to build your cube and query it. The username/password is ADMIN/KYLIN
+Kylin Rquires a properly setup hadoop environment to run. Please take a look [this](Hadoop Environment.md).
-### Build Cube in Cubes Tab
+### Install Kylin
-
-### Check Building Progress in Job Tab
-
-### Query Tables in Query Tab
-
+1. Download latest Kylin binaries at http://kylin.incubator.apache.org/download/
+2. export KYLIN_HOME pointing to the extracted Kylin folder
+3. Make sure the user has the privilege to run hadoop, hive and hbase cmd in shell. If you are not so sure, you can just run **bin/check-env.sh**, it will print out the detail information if you have some environment issues.
+4. To start Kylin, simply run **bin/kylin.sh start**
+5. To stop Kylin, simply run **bin/kylin.sh stop**
-## What if I closed my VM?
+> If you want to have multiple Kylin instances please refer to [this](Multiple Kylin REST servers.md)
-If you shut down the VM and restarted it, Kylin will not automatically start. Depending on whether you succeed in running the deploy.sh, you should:
+After Kylin started you can visit <http://your_sandbox_ip:7070/kylin>. The username/password is ADMIN/KYLIN. It's a clean Kylin homepage with nothing in there. To start with you can:
-### If running deploy.sh failed last time
-1. Kill it if any tomcat instance exist
-2. Start over again
+1. [Quick play with a sample cube](../Tutorial/Quick play with a sample cube.md)
+2. [Create and Build your own cube](../Tutorial/Kylin Cube Creation Tutorial.md)
-### If running deploy.sh succeeded last time
-1. Kill it if any tomcat instance exist
-2. Make sure Hbase is running
-3. run `export CATALINA_HOME=~/apache-tomcat-7.0.56`
-4. run `sudo -i "${CATALINA_HOME}/bin/startup.sh"`
\ No newline at end of file
+Here's also a overview introduction on the website [Kylin Web Tutorial](../Tutorial/Kylin Web Tutorial.md)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/On Hadoop CLI installation.pdf
----------------------------------------------------------------------
diff --git a/docs/Installation/On Hadoop CLI installation.pdf b/docs/Installation/On Hadoop CLI installation.pdf
deleted file mode 100644
index 636555a..0000000
Binary files a/docs/Installation/On Hadoop CLI installation.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/On Hadoop Kylin installation using Docker.md
----------------------------------------------------------------------
diff --git a/docs/Installation/On Hadoop Kylin installation using Docker.md b/docs/Installation/On Hadoop Kylin installation using Docker.md
index 81b2d7f..694b738 100644
--- a/docs/Installation/On Hadoop Kylin installation using Docker.md
+++ b/docs/Installation/On Hadoop Kylin installation using Docker.md
@@ -36,5 +36,5 @@ Use `admin/admin` to login. Make sure HBase is running.
#Kylin Dashboard
http://<container_ip>:7070
```
-The default credentials to login are: `ADMIN:KYLIN`. The cluster is pre-populated with sample data and is ready to build cubes as shown [here](https://github.com/KylinOLAP/Kylin/wiki/Kylin-Cube-Creation-Tutorial).
+The default credentials to login are: `ADMIN:KYLIN`. The cluster is pre-populated with sample data and is ready to build cubes as shown [here](../Tutorial/Kylin Cube Build and Job Monitoring Tutorial.md).
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/On Hadoop Kylin installation using Docker.pdf
----------------------------------------------------------------------
diff --git a/docs/Installation/On Hadoop Kylin installation using Docker.pdf b/docs/Installation/On Hadoop Kylin installation using Docker.pdf
deleted file mode 100644
index a30fe15..0000000
Binary files a/docs/Installation/On Hadoop Kylin installation using Docker.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/install_and_run.md
----------------------------------------------------------------------
diff --git a/docs/Installation/install_and_run.md b/docs/Installation/install_and_run.md
deleted file mode 100644
index 16179fa..0000000
--- a/docs/Installation/install_and_run.md
+++ /dev/null
@@ -1,83 +0,0 @@
-##Install and Run
-
-###How to run
-
-1. Download the release version(according to the hadoop distribution)
-2. Setup a KYLIN_HOME pointing to the corresponding directory where you extract the release tar
-3. Make sure the user has the privilege to run hadoop, hive and hbase cmd in shell. If you are not so sure, you can just run **bin/check-env.sh**, it will print out the detail information if you have some environment issues.
-4. To start Kylin, simply run **bin/kylin.sh start**
-5. To stop Kylin, simply run **bin/kylin.sh stop**
-
-
-If you are running Kylin in a cluster or you have multiple Kylin instances, please make sure you have the following property correctly configured.
-
-1. kylin.rest.servers
-
- List of web servers in use, this enables one web server instance to sync up with other servers.
-
-
-2. kylin.server.mode
-
- Make sure there is only one instance whose "kylin.server.mode" is set to "all" if there are multiple instances.
-
-
-###The directory structure of the installation
-
-> ├── bin
-> │ ├── check-env.sh
-> │ ├── find-hive-dependency.sh
-> │ ├── health-check.sh
-> │ ├── kylin.sh
-> │ └── sample.sh
-> │
-> ├── conf
-> │ ├── kylin_job_conf.xml
-> │ └── kylin.properties
-> │
-> ├── sample_cube
-> │ ├── data
-> │ └── metadata
-> │
-> ├── tomcat
-> │ ├── webapps
-> │ │ └── kylin.war
-> │ …….
-> │
-> └── lib
-> ├── kylin-coprocessor-${version}.jar
-> ├── kylin-job-${version}.jar
-> └── kylin-jdbc-${version}.jar
-
-###Create sample cube
-Kylin provides a script for you to create a sample Cube; the script will also create three sample hive tables:
-
-1. Run ${KYLIN_HOME}/bin/sample.sh
-2. Logon Kylin web, select project "learn_kylin";
-3. Select the sample cube "kylin_sales_cube", click "Actions" -> "Build", pick up a date later than 2014-01-01 (to cover all 10000 sample records);
-4. Check the build progress in "Jobs" tab, until 100%;
-5. Execute SQLs in the "Query" tab, for example:
- select cal_dt, sum(price) as total_selled, count(distinct seller_id) as sellers from kylin_sales group by cal_dt order by cal_dt
-6. You can verify the query result and compare the response time with hive;
-
-###Build from source
-> git clone https://github.com/KylinOLAP/Kylin.git
-> cd KylinOLAP/Kylin
-> sh script/package.sh
-
-In order to generate binary package, **maven** and **npm** are pre-requisites.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/off_cli_install_scene.png
----------------------------------------------------------------------
diff --git a/docs/Installation/off_cli_install_scene.png b/docs/Installation/off_cli_install_scene.png
new file mode 100644
index 0000000..26e9e4f
Binary files /dev/null and b/docs/Installation/off_cli_install_scene.png differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Installation/on_cli_install_scene.png
----------------------------------------------------------------------
diff --git a/docs/Installation/on_cli_install_scene.png b/docs/Installation/on_cli_install_scene.png
new file mode 100644
index 0000000..5b07d25
Binary files /dev/null and b/docs/Installation/on_cli_install_scene.png differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/README.md
----------------------------------------------------------------------
diff --git a/docs/README.md b/docs/README.md
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin Cube Build and Job Monitoring Tutorial.pdf
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin Cube Build and Job Monitoring Tutorial.pdf b/docs/Tutorial/Kylin Cube Build and Job Monitoring Tutorial.pdf
deleted file mode 100644
index b4253d4..0000000
Binary files a/docs/Tutorial/Kylin Cube Build and Job Monitoring Tutorial.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin Cube Creation Tutorial.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin Cube Creation Tutorial.md b/docs/Tutorial/Kylin Cube Creation Tutorial.md
index cc85075..294f665 100644
--- a/docs/Tutorial/Kylin Cube Creation Tutorial.md
+++ b/docs/Tutorial/Kylin Cube Creation Tutorial.md
@@ -119,4 +119,11 @@ Choose partition type, partition column and start date.
You can overview your cube and go back to previous step to modify it. Click the `Save` button to complete the cube creation.
-
\ No newline at end of file
+
+
+### IV. What's next
+
+After cube being created, you might need to:
+
+1. [Build the cube so that it can be queried](Kylin Cube Build and Job Monitoring Tutorial.md)
+2. [Grant permission to cubes](Kylin Cube Permission Grant Tutorial.md)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin Cube Creation Tutorial.pdf
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin Cube Creation Tutorial.pdf b/docs/Tutorial/Kylin Cube Creation Tutorial.pdf
deleted file mode 100644
index cd6e934..0000000
Binary files a/docs/Tutorial/Kylin Cube Creation Tutorial.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin Cube Permission Grant Tutorial.pdf
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin Cube Permission Grant Tutorial.pdf b/docs/Tutorial/Kylin Cube Permission Grant Tutorial.pdf
deleted file mode 100644
index adfb6d7..0000000
Binary files a/docs/Tutorial/Kylin Cube Permission Grant Tutorial.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin ODBC Driver Tutorial.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin ODBC Driver Tutorial.md b/docs/Tutorial/Kylin ODBC Driver Tutorial.md
index 43439fc..7f84213 100644
--- a/docs/Tutorial/Kylin ODBC Driver Tutorial.md
+++ b/docs/Tutorial/Kylin ODBC Driver Tutorial.md
@@ -26,5 +26,12 @@ Kylin ODBC Driver Tutorial
3. Both drivers already be installed on Tableau Server, you properly should be able to publish to there without issues
+## Use ODBC compatible applications
+
+1. [Kylin and Tableau Tutorial](Kylin and Tableau Tutorial.md)
+2. [Kylin and Excel Tutorial](Kylin and Excel Tutorial.md)
+
+
+
## Bug Report
Open github issue here
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin ODBC Driver Tutorial.pdf
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin ODBC Driver Tutorial.pdf b/docs/Tutorial/Kylin ODBC Driver Tutorial.pdf
deleted file mode 100644
index e2d4986..0000000
Binary files a/docs/Tutorial/Kylin ODBC Driver Tutorial.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin Web Tutorial.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin Web Tutorial.md b/docs/Tutorial/Kylin Web Tutorial.md
index 5328370..af49d9b 100644
--- a/docs/Tutorial/Kylin Web Tutorial.md
+++ b/docs/Tutorial/Kylin Web Tutorial.md
@@ -8,7 +8,7 @@ Kylin Web Tutorial
> Mac: Google Chrome, FireFox, Safari
## 1. Access & Login
-Host to access: http://your_sandbox_ip:9080
+Host to access: http://your_sandbox_ip:7070/kylin
Login with username/password: ADMIN/KYLIN
@@ -19,6 +19,9 @@ Although Kylin will using SQL as query interface and leverage Hive metadata, kyl
## 3. Kylin OLAP Cube
+
+> To make cubes availabe you'll have to create them first [Kylin Cube Creation Tutorial](Kylin Cube Creation Tutorial.md)
+
Kylin's OLAP Cubes are pre-calculation datasets from Star Schema Hive tables, Here's the web management interface for user to explorer, manage all cubes.Go to `Cubes` Menu, it will list all cubes available in system:
@@ -42,6 +45,9 @@ To explore more detail about the Cube
## 4. Write and Execute SQL on web
+
+>> To make queries availabe you'll have to create are build cubes first [Kylin Cube Creation Tutorial](Kylin Cube Creation Tutorial.md) and [Kylin Cube Build and Job Monitoring Tutorial](Kylin Cube Build and Job Monitoring Tutorial.md)
+
Kylin's web offer a simple query tool for user to run SQL to explorer existing cube, verify result and explorer the result set using #5's Pivot analysis and visualization
> **Query Limit**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin Web Tutorial.pdf
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin Web Tutorial.pdf b/docs/Tutorial/Kylin Web Tutorial.pdf
deleted file mode 100644
index 6f27a4b..0000000
Binary files a/docs/Tutorial/Kylin Web Tutorial.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin and Excel Tutorial.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin and Excel Tutorial.md b/docs/Tutorial/Kylin and Excel Tutorial.md
new file mode 100644
index 0000000..49961e0
--- /dev/null
+++ b/docs/Tutorial/Kylin and Excel Tutorial.md
@@ -0,0 +1,3 @@
+Kylin and Excel Tutorial
+===
+For now our ODBC driver has not yet finished supporting Excel, watch the mail list for updates.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Kylin and Tableau Tutorial.pdf
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin and Tableau Tutorial.pdf b/docs/Tutorial/Kylin and Tableau Tutorial.pdf
deleted file mode 100644
index 14edbe2..0000000
Binary files a/docs/Tutorial/Kylin and Tableau Tutorial.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/docs/Tutorial/Quick play with a sample cube.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Quick play with a sample cube.md b/docs/Tutorial/Quick play with a sample cube.md
new file mode 100644
index 0000000..cd767b3
--- /dev/null
+++ b/docs/Tutorial/Quick play with a sample cube.md
@@ -0,0 +1,10 @@
+### Quick start with sample cube
+Kylin provides a script for you to create a sample Cube; the script will also create three sample hive tables:
+
+1. Run ${KYLIN_HOME}/bin/sample.sh
+2. Logon Kylin web, select project "learn_kylin";
+3. Select the sample cube "kylin_sales_cube", click "Actions" -> "Build", pick up a date later than 2014-01-01 (to cover all 10000 sample records);
+4. Check the build progress in "Jobs" tab, until 100%;
+5. Execute SQLs in the "Query" tab, for example:
+ select cal_dt, sum(price) as total_selled, count(distinct seller_id) as sellers from kylin_sales group by cal_dt order by cal_dt
+6. You can verify the query result and compare the response time with hive;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/592e96b5/old_README.md
----------------------------------------------------------------------
diff --git a/old_README.md b/old_README.md
new file mode 100644
index 0000000..44f6859
--- /dev/null
+++ b/old_README.md
@@ -0,0 +1,94 @@
+Kylin OLAP
+===========
+
+
+Overview
+------------
+> Extreme OLAP Engine for Big Data
+
+An open source distributed Analytical Engine from eBay to provide SQL interface and multi-dimensional analysis (OLAP) on Hadoop to support TB to PB size analysis.
+
+
+Getting Started
+---------------
+
+### Prerequisites ###
+
+#### Hadoop ####
+- Hadoop: 2.2.0.2.0.6.0-61 or above
+- Hive: 0.12.0.2.0.6.0-61 or above
+- HBase: 0.96.0.2.0.6.0-61-hadoop2
+
+
+_Tested with Hortonworks HDP 2.1.3 and Cloudera Quickstart VM 5.1._
+
+#### Misc ####
+- Command hadoop, hive, hbase is workable on your hadoop cluster
+- JDK Runtime: JDK7 (OpenJDK or Oracle JDK)
+- Maven
+- Git
+- Tomcat (CATALINA_HOME being set)
+- Npm
+
+#### Before Install ####
+- For the quick tutorial, we assume that your hadoop has priviledges disabled which means any user could run hadoop and hive command
+
+
+Installation
+------------
+It is very easy to install Kylin for exploration/development. There are 3 supported options:
+
+
+1. Sandbox (HDP or CDH sandbox)
+2. Dev Environment (IDE + Sandbox)
+3. Docker container (Docker)
+
+#### Sandbox (On-Hadoop-CLI installation) ####
+
+If you are free to install Kylin on your hadoop CLI machine or Hadoop sandbox, this is the most convenient scenario, for it puts everything in a single machine.
+
+
+
+
+
+For normal users, the application in the above picture means `Kylin Web`, which contains a web interface for cube building, querying and all sorts of management. Kylin Web launches a query engine for querying and a cube build engine for building cubes. These two engines interact with the components in Hadoop CLI, like hive and hbase.
+
+For a hands-on tutorial please visit [On-Hadoop-CLI installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-CLI-installation).
+
+#### Dev Environment (Off-Hadoop-CLI Installation) ####
+
+This is typically for development environment setup.
+
+Applications does not necessarily mean Kylin Web, it can also be any test cases or java applications running at your local IDE(query engine and cube build engine can be launched programmatically in your code). In this case, it is no longer reasonable to assume that all the Kylin components reside in the same machine as your Hadoop CLI. Fortunately, Kylin still works under such condition with proper settings on your CLI machine.
+
+
+
+For a hands-on tutorial please visit [Off-Hadoop-CLI installation](https://github.com/KylinOLAP/Kylin/wiki/Off-Hadoop-CLI-Installation-(Dev-Env-Setup))
+
+#### Docker Container ####
+With help from [SequenceIQ](http://sequenceiq.com/), there's docker container for Kylin (along with Hadoop, HBase and Hive) available now:[sequenceiq/docker-kylin](https://github.com/sequenceiq/docker-kylin). The only thing you will need to do is to pull the container from the official Docker repository to be up and running in few minutes.
+
+Features:
+
+- Size - Light weight compared to downloading and setting up HDP or CDH sandbox.
+- Cluster support - Supports multi node installation.
+- Fully Automated - No manual steps. One command does it all
+
+For a hands-on tutorial please visit [Kylin Docker installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-Kylin-installation-using-Docker).
+
+
+
+### Resources ###
+
+* Web Site: <http://kylin.io>
+
+* Google Group: [Kylin OLAP Group](https://groups.google.com/forum/#!forum/kylin-olap)
+
+* Developer Mail: <ky...@googlegroups.com>
+
+* How To Contribute: See [wiki](https://github.com/KylinOLAP/Kylin/wiki/How-to-Contribute)
+
+* Presentation: [Kylin Hadoop OLAP Engine v1.0](https://github.com/KylinOLAP/Kylin/blob/master/docs/Kylin_Hadoop_OLAP_Engine_v1.0.pdf?raw=true)
+
+* Apache Proposal: [Apache Kylin](https://wiki.apache.org/incubator/KylinProposal)
+
[42/50] [abbrv] incubator-kylin git commit: Merge branch 'staging' of
https://github.com/KylinOLAP/Kylin into staging
Posted by li...@apache.org.
Merge branch 'staging' of https://github.com/KylinOLAP/Kylin into staging
Conflicts:
query/src/test/resources/query/sql/query78.sql
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/3655b338
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/3655b338
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/3655b338
Branch: refs/heads/streaming-localdict
Commit: 3655b338f9833a894fc873a393e294dcb7a3cf78
Parents: 222b144 3563a91
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 20 15:45:37 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 20 15:45:37 2015 +0800
----------------------------------------------------------------------
----------------------------------------------------------------------
[16/50] [abbrv] incubator-kylin git commit: Build cube in mem
Posted by li...@apache.org.
Build cube in mem
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dcc72176
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dcc72176
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dcc72176
Branch: refs/heads/streaming-localdict
Commit: dcc721769ca0fdf88c270f81043fbbd7b2a1cb29
Parents: 5925320
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Thu Mar 19 18:13:45 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Thu Mar 19 18:13:45 2015 +0800
----------------------------------------------------------------------
.../kylin/job/BuildCubeWithStreamTest.java | 147 ++++++++
streaming/pom.xml | 14 +-
.../kylin/streaming/cube/CubeStreamBuilder.java | 360 +++++++++++++++++++
3 files changed, 519 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dcc72176/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
new file mode 100644
index 0000000..1e25b69
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -0,0 +1,147 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.streaming.Stream;
+import org.apache.kylin.streaming.cube.CubeStreamBuilder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Created by qianzhou on 3/9/15.
+ */
+public class BuildCubeWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithStreamTest.class);
+
+ private KylinConfig kylinConfig;
+ private CubeManager cubeManager;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ cubeManager = CubeManager.getInstance(kylinConfig);
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_empty");
+ final CubeDesc desc = cube.getDescriptor();
+ cube.getSegments().clear();
+ cubeManager.updateCube(cube);
+
+
+// final String tableName = createIntermediateTable(desc, kylinConfig, null);
+ String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af";
+ logger.info("intermediate table name:" + tableName);
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+
+ LinkedBlockingDeque<Stream> queue = new LinkedBlockingDeque<Stream>();
+
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ final CubeStreamBuilder streamBuilder = new CubeStreamBuilder(queue, "", desc, 0);
+ while (reader.next()) {
+ queue.put(parse(reader.getRow()));
+ }
+ queue.put(new Stream(-1, null));
+ final Future<?> future = executorService.submit(streamBuilder);
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ fail("stream build failed");
+ }
+
+ logger.info("stream build finished, htable name:");
+ }
+
+
+ private void clearSegment(String cubeName) throws Exception {
+ CubeInstance cube = cubeManager.getCube(cubeName);
+ cube.getSegments().clear();
+ cubeManager.updateCube(cube);
+ }
+
+
+ private Stream parse(String[] row) {
+ return new Stream(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dcc72176/streaming/pom.xml
----------------------------------------------------------------------
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 62880eb..8559c92 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -18,9 +18,19 @@
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-invertedindex</artifactId>
- <version>0.7.1-SNAPSHOT</version>
+ <version>${project.parent.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-cube</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-storage</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
-
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dcc72176/streaming/src/main/java/org/apache/kylin/streaming/cube/CubeStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/cube/CubeStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/cube/CubeStreamBuilder.java
new file mode 100644
index 0000000..0ae9a5e
--- /dev/null
+++ b/streaming/src/main/java/org/apache/kylin/streaming/cube/CubeStreamBuilder.java
@@ -0,0 +1,360 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+package org.apache.kylin.streaming.cube;
+
+import com.google.common.base.Function;
+import com.google.common.collect.*;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.metadata.measure.MeasureAggregators;
+import org.apache.kylin.metadata.measure.MeasureCodec;
+import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.gridtable.*;
+import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
+import org.apache.kylin.streaming.Stream;
+import org.apache.kylin.streaming.StreamBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.LinkedBlockingDeque;
+
+/**
+ * Created by shaoshi on 3/12/2015.
+ */
+public class CubeStreamBuilder extends StreamBuilder {
+
+ private static Logger logger = LoggerFactory.getLogger(CubeStreamBuilder.class);
+
+ private CubeDesc desc = null;
+ private int partitionId = -1;
+ CuboidScheduler cuboidScheduler = null;
+ List<List<String>> table = null;
+ Map<TblColRef, Dictionary<?>> dictionaryMap = null;
+ private Cuboid baseCuboid = null;
+ private CubeInstance cube;
+ private CubeSegment cubeSegment = null;
+ private CubeJoinedFlatTableDesc intermediateTableDesc;
+ private MeasureCodec measureCodec;
+ private MeasureAggregators aggs = null;
+ private int measureNumber;
+
+ private String[] metricsAggrFuncs = null;
+
+ public CubeStreamBuilder(LinkedBlockingDeque<Stream> queue, String hTableName, CubeDesc desc, int partitionId) {
+ super(queue, 2000);
+ this.desc = desc;
+ this.partitionId = partitionId;
+ this.cuboidScheduler = new CuboidScheduler(desc);
+
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ List<CubeInstance> cubes = cubeManager.getCubesByDesc(this.desc.getName());
+ cube = cubes.get(0);
+
+ measureCodec = new MeasureCodec(desc.getMeasures());
+ aggs = new MeasureAggregators(desc.getMeasures());
+ measureNumber = desc.getMeasures().size();
+
+ List<String> metricsAggrFuncsList = Lists.newArrayList();
+ for (int i = 0, n = desc.getMeasures().size(); i < n; i++) {
+ MeasureDesc measureDesc = desc.getMeasures().get(i);
+ metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
+ }
+ metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
+ }
+
+ @Override
+ protected void build(List<Stream> streamsToBuild) {
+
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ try {
+ cubeSegment = cubeManager.appendSegments(cube, System.currentTimeMillis());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), cubeSegment);
+
+ table = Lists.transform(streamsToBuild, new Function<Stream, List<String>>() {
+ @Nullable
+ @Override
+ public List<String> apply(@Nullable Stream input) {
+ return parseStream(input, desc);
+ }
+ });
+
+ // dictionaryMap = buildDictionary(table, desc);
+
+ long baseCuboidId = Cuboid.getBaseCuboidId(desc);
+
+ List<GridTable> result = new LinkedList<GridTable>();
+ try {
+ calculateCuboid(null, -1l, baseCuboidId, result);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ assert result.size() > 0;
+ }
+
+
+ private void calculateCuboid(GridTable parentCuboid, long parentCuboidId, long cuboidId, List<GridTable> result) throws IOException {
+
+ GridTable thisCuboid;
+ if (parentCuboidId < 0) {
+ thisCuboid = calculateBaseCuboid(this.table, cuboidId);
+ } else {
+ thisCuboid = aggregateCuboid(parentCuboid, parentCuboidId, cuboidId);
+ }
+
+ ArrayList<Long> children = (ArrayList<Long>) cuboidScheduler.getSpanningCuboid(cuboidId);
+ Collections.sort(children);
+ // making sure the children are sorted in ascending order
+ for (Long childId : children) {
+ calculateCuboid(thisCuboid, cuboidId, childId, result);
+ }
+
+ result.add(thisCuboid);
+ }
+
+
+ private GridTable calculateBaseCuboid(List<List<String>> table, long baseCuboidId) throws IOException {
+
+ GridTable gridTable = newGridTableByCuboidID(baseCuboidId);
+ GTRecord r = new GTRecord(gridTable.getInfo());
+ GTBuilder builder = gridTable.rebuild();
+
+ for (List<String> row : table) {
+ String[] dimensions = buildKey(row);
+ Object[] metricsValues = buildValue(row);
+ Object[] recordValues = new Object[dimensions.length + metricsValues.length];
+ System.arraycopy(dimensions, 0, recordValues, 0, dimensions.length);
+ System.arraycopy(metricsValues, 0, recordValues, dimensions.length, metricsValues.length);
+ builder.write(r.setValues(recordValues));
+ }
+ builder.close();
+ Pair<BitSet, BitSet> dimensionMetricsBitSet = getDimensionAndMetricColumBitSet(baseCuboidId);
+
+ IGTScanner scanner = gridTable.scanAndAggregate(null, null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
+
+
+ GridTable baseCuboidGridTable = newGridTableByCuboidID(baseCuboidId);
+ builder = baseCuboidGridTable.rebuild();
+ for (GTRecord record : scanner) {
+ builder.write(record);
+ }
+ builder.close();
+
+ return baseCuboidGridTable;
+ }
+
+ private GridTable newGridTableByCuboidID(long cuboidID) {
+ GTInfo info = newGTInfo(cuboidID);
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable gridTable = new GridTable(info, store);
+
+ return gridTable;
+ }
+
+
+ private GridTable aggregateCuboid(GridTable parentCuboid, long parentCuboidId, long cuboidId) throws IOException {
+ Pair<BitSet, BitSet> columnBitSets = getDimensionAndMetricColumBitSet(parentCuboidId);
+
+ BitSet dimensionBitSet = columnBitSets.getFirst();
+ long mask = Long.highestOneBit(parentCuboidId);
+ long childCuboidId = cuboidId;
+ long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(parentCuboidId);
+ int index = 0;
+ for (int i = 0; i < parentCuboidIdActualLength; i++) {
+ if ((mask & parentCuboidId) > 0) {
+ if ((mask & childCuboidId) == 0) {
+ // this dim will be aggregated
+ dimensionBitSet.set(index, false);
+ }
+ index++;
+ }
+ mask = mask >> 1;
+ }
+
+
+ IGTScanner scanner = parentCuboid.scanAndAggregate(null, null, dimensionBitSet, columnBitSets.getSecond(), metricsAggrFuncs, null);
+ GridTable cuboidGridTable = newGridTableByCuboidID(cuboidId);
+ GTBuilder builder = cuboidGridTable.rebuild();
+ for (GTRecord record : scanner) {
+ builder.write(record);
+ }
+ builder.close();
+
+
+ return cuboidGridTable;
+ }
+
+ private Pair<BitSet, BitSet> getDimensionAndMetricColumBitSet(long cuboidId) {
+ BitSet bitSet = BitSet.valueOf(new long[]{cuboidId});
+ BitSet dimension = new BitSet();
+ dimension.set(0, bitSet.cardinality());
+ BitSet metrics = new BitSet();
+ metrics.set(bitSet.cardinality(), bitSet.cardinality() + this.measureNumber);
+ return new Pair<BitSet, BitSet>(
+ dimension, metrics
+ );
+ }
+
+ private void aggregate(Object[] existing, Object[] newRow) {
+ aggs.reset();
+ aggs.aggregate(existing);
+ aggs.aggregate(newRow);
+
+ aggs.collectStates(existing);
+ }
+
+ private String[] buildKey(List<String> row) {
+ int keySize = intermediateTableDesc.getRowKeyColumnIndexes().length;
+ String[] key = new String[keySize];
+
+ for (int i = 0; i < keySize; i++) {
+ key[i] = row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]);
+ }
+
+ return key;
+ }
+
+
+ private Object[] buildValue(List<String> row) {
+
+ Object[] values = new Object[desc.getMeasures().size()];
+ MeasureDesc measureDesc = null;
+ for (int i = 0, n = desc.getMeasures().size(); i < n; i++) {
+ measureDesc = desc.getMeasures().get(i);
+ Object value = null;
+ int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[i];
+ if (flatTableIdx == null) {
+ value = measureCodec.getSerializer(i).valueOf(measureDesc.getFunction().getParameter().getValue());
+ } else if (flatTableIdx.length == 1) {
+ value = measureCodec.getSerializer(i).valueOf(Bytes.toBytes(row.get(flatTableIdx[0])));
+ } else {
+ for (int x : flatTableIdx) {
+ value = value + row.get(x);
+ }
+ }
+ values[i] = value;
+ }
+
+ return values;
+ }
+
+
+ private GTInfo newGTInfo(long cuboidID) {
+ Pair<BitSet, BitSet> dimensionMetricsBitSet = getDimensionAndMetricColumBitSet(cuboidID);
+ GTInfo.Builder builder = infoBuilder(cuboidID);
+ builder.enableColumnBlock(new BitSet[]{dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond()});
+ builder.setPrimaryKey(dimensionMetricsBitSet.getFirst());
+ GTInfo info = builder.build();
+ return info;
+ }
+
+ private GTInfo.Builder infoBuilder(long cuboidID) {
+ Cuboid cuboid = Cuboid.findById(desc, cuboidID);
+
+ List<DataType> dataTypes = new ArrayList<DataType>(cuboid.getColumns().size() + this.measureNumber);
+ for (TblColRef col : cuboid.getColumns()) {
+ dataTypes.add(col.getType());
+ }
+ for (MeasureDesc measure : this.desc.getMeasures()) {
+ dataTypes.add(measure.getFunction().getReturnDataType());
+ }
+
+ GTInfo.Builder builder = GTInfo.builder();
+ builder.setCodeSystem(new GTSampleCodeSystem());
+ builder.setColumns(dataTypes.toArray(new DataType[dataTypes.size()]));
+ return builder;
+ }
+
+
+ private Map<TblColRef, Dictionary<?>> buildDictionary(List<List<String>> table, CubeDesc desc) {
+ SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
+
+ List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
+ for (List<String> row : table) {
+ for (int i = 0; i < dimColumns.size(); i++) {
+ String cell = row.get(i);
+ valueMap.put(dimColumns.get(i), cell);
+ }
+ }
+ Map<TblColRef, Dictionary<?>> result = Maps.newHashMap();
+
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ logger.info("Building dictionary for " + col);
+ result.put(col, DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
+ @Nullable
+ @Override
+ public byte[] apply(String input) {
+ return input.getBytes();
+ }
+ })));
+ }
+ }
+ }
+
+ return result;
+ }
+
+ private List<String> parseStream(Stream stream, CubeDesc desc) {
+ return Lists.newArrayList(new String(stream.getRawData()).split(","));
+ }
+
+
+ private void submitOffset() {
+
+ }
+
+}
[08/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/db3cb94f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/db3cb94f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/db3cb94f
Branch: refs/heads/streaming-localdict
Commit: db3cb94f88997ec731e4a2b93569e17082f4ce24
Parents: 05c2d09 2695345
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Mar 18 18:23:06 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Mar 18 18:23:06 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/DictionaryGenerator.java | 19 ++-
.../apache/kylin/invertedindex/index/Slice.java | 2 +-
.../invertedindex/index/TableRecordInfo.java | 42 ++---
.../index/TableRecordInfoDigest.java | 11 +-
.../invertedindex/model/IIKeyValueCodec.java | 160 +++++--------------
.../apache/kylin/job/BuildIIWithStreamTest.java | 8 +-
.../hbase/InvertedIndexStorageEngine.java | 6 +-
.../endpoint/EndpointTupleIterator.java | 9 +-
.../hbase/coprocessor/endpoint/IIEndpoint.java | 12 +-
.../invertedindex/IIStreamBuilder.java | 11 +-
10 files changed, 101 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
[34/50] [abbrv] incubator-kylin git commit: KYLIN-650 add more
convensions
Posted by li...@apache.org.
KYLIN-650 add more convensions
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0be89306
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0be89306
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0be89306
Branch: refs/heads/streaming-localdict
Commit: 0be893069d7e4fae7dcaf4b85ce839f930ce93d7
Parents: a8757d5
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 23:38:56 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 23:38:56 2015 -0700
----------------------------------------------------------------------
docs/MISC/Kylin Document Convensions.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0be89306/docs/MISC/Kylin Document Convensions.md
----------------------------------------------------------------------
diff --git a/docs/MISC/Kylin Document Convensions.md b/docs/MISC/Kylin Document Convensions.md
index 0d45dd9..52fa6f7 100644
--- a/docs/MISC/Kylin Document Convensions.md
+++ b/docs/MISC/Kylin Document Convensions.md
@@ -5,4 +5,5 @@ Kylin Document Convernsions
2. In markdown files, try to use relative URI for document/image reference.
3. Never link to github wiki pages
4. Try not to simply list all docuemnts in one page. Instead, try to organize different documents into a "story", so that readers are presented deeper materials as they proceed.
-5. ...
\ No newline at end of file
+5. Always keep documents updated
+6. ...
\ No newline at end of file
[35/50] [abbrv] incubator-kylin git commit: KYLIN-650 typo
Posted by li...@apache.org.
KYLIN-650 typo
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/269eac01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/269eac01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/269eac01
Branch: refs/heads/streaming-localdict
Commit: 269eac01c35489727ba6ec509b5449d9d4c8dcde
Parents: 0be8930
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 23:53:26 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 23:53:26 2015 -0700
----------------------------------------------------------------------
docs/MISC/Kylin Document Convensions.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/269eac01/docs/MISC/Kylin Document Convensions.md
----------------------------------------------------------------------
diff --git a/docs/MISC/Kylin Document Convensions.md b/docs/MISC/Kylin Document Convensions.md
index 52fa6f7..a5b4fa0 100644
--- a/docs/MISC/Kylin Document Convensions.md
+++ b/docs/MISC/Kylin Document Convensions.md
@@ -1,4 +1,4 @@
-Kylin Document Convernsions
+Kylin Document Convensions
-----
1. Keep documentation in markdown files
[49/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ee2bb66d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ee2bb66d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ee2bb66d
Branch: refs/heads/streaming-localdict
Commit: ee2bb66dc8df4e3abd2fac87ff113ecacb4bb62f
Parents: 21909b9 d4d0675
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 20 18:48:03 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 20 18:48:03 2015 +0800
----------------------------------------------------------------------
.../metadata/measure/MeasureAggregator.java | 2 +-
.../apache/kylin/storage/gridtable/GTInfo.java | 2 +-
.../storage/gridtable/GTSampleCodeSystem.java | 2 +-
.../kylin/streaming/cube/CubeStreamBuilder.java | 95 +++++++++++++-------
4 files changed, 64 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
[21/50] [abbrv] incubator-kylin git commit: KYLIN-650 updates
Posted by li...@apache.org.
KYLIN-650 updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/da16316a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/da16316a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/da16316a
Branch: refs/heads/streaming-localdict
Commit: da16316a54f6f04c91ad62d104bea6844c533a49
Parents: 592e96b
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 19:04:45 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 19:04:45 2015 -0700
----------------------------------------------------------------------
docs/Installation/Installation.md | 2 +-
docs/Installation/Off Hadoop CLI Installation.md | 2 +-
docs/Tutorial/Quick play with a sample cube.md | 10 +++++++++-
3 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/da16316a/docs/Installation/Installation.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Installation.md b/docs/Installation/Installation.md
index 5b33a56..2fa345c 100644
--- a/docs/Installation/Installation.md
+++ b/docs/Installation/Installation.md
@@ -32,7 +32,7 @@ Features:
- Cluster support - Supports multi node installation.
- Fully Automated - No manual steps. One command does it all
-For a hands-on tutorial please visit [Kylin Docker installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-Kylin-installation-using-Docker).
+For a hands-on tutorial please visit [Kylin Docker installation](On Hadoop Kylin installation using Docker.md).
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/da16316a/docs/Installation/Off Hadoop CLI Installation.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Off Hadoop CLI Installation.md b/docs/Installation/Off Hadoop CLI Installation.md
index d63bf13..f83ed5d 100644
--- a/docs/Installation/Off Hadoop CLI Installation.md
+++ b/docs/Installation/Off Hadoop CLI Installation.md
@@ -63,5 +63,5 @@ Run other tests, the end-to-end cube building test is exclueded
In your Eclipse IDE, launch `org.apache.kylin.rest.DebugTomcat` with specifying VM arguments "-Dspring.profiles.active=sandbox". (By default Kylin server will listen on 7070 port; If you want to use another port, please specify it as a parameter when run `DebugTomcat)
-Check Kylin Web available at http://localhost:7070 (user:ADMIN,password:KYLIN)
+Check Kylin Web available at http://localhost:7070/kylin (user:ADMIN,password:KYLIN)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/da16316a/docs/Tutorial/Quick play with a sample cube.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Quick play with a sample cube.md b/docs/Tutorial/Quick play with a sample cube.md
index cd767b3..37c44ff 100644
--- a/docs/Tutorial/Quick play with a sample cube.md
+++ b/docs/Tutorial/Quick play with a sample cube.md
@@ -1,4 +1,5 @@
### Quick start with sample cube
+
Kylin provides a script for you to create a sample Cube; the script will also create three sample hive tables:
1. Run ${KYLIN_HOME}/bin/sample.sh
@@ -7,4 +8,11 @@ Kylin provides a script for you to create a sample Cube; the script will also cr
4. Check the build progress in "Jobs" tab, until 100%;
5. Execute SQLs in the "Query" tab, for example:
select cal_dt, sum(price) as total_selled, count(distinct seller_id) as sellers from kylin_sales group by cal_dt order by cal_dt
-6. You can verify the query result and compare the response time with hive;
\ No newline at end of file
+6. You can verify the query result and compare the response time with hive;
+
+#### What's next
+
+After cube being created, you might need to:
+
+1. [Build the cube so that it can be queried](Kylin Cube Build and Job Monitoring Tutorial.md)
+2. [Grant permission to cubes](Kylin Cube Permission Grant Tutorial.md)
\ No newline at end of file
[28/50] [abbrv] incubator-kylin git commit: KYLIN-650 minor change
Posted by li...@apache.org.
KYLIN-650 minor change
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4bd54078
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4bd54078
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4bd54078
Branch: refs/heads/streaming-localdict
Commit: 4bd54078af6dd7c60c4bb5eb8aab4a99e2a6d5f3
Parents: eff1393
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 22:27:23 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 22:27:23 2015 -0700
----------------------------------------------------------------------
README.md | 12 ++++--
...requently Asked Questions on Installation.md | 26 ------------
.../MISC/FAQ on Kylin Installation and Usage.md | 40 +++++++++++++++++++
docs/MISC/How to Contribute.md | 42 ++++++++++++++++++++
4 files changed, 90 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4bd54078/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 232758c..2489576 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,10 @@ Tutorial
Please follow this installation tutorial to start with Kylin: [Installation Tutorial](docs/Installation/Installation.md)
+Advance Usage
+-------
+
+
Get Help
------------
@@ -26,13 +30,13 @@ The fastest way to get response from our developers is to send email to our mail
Resources
------------
-* Web Site: <http://kylin.io>
+* [FAQ](docs/MISC/FAQ on Kylin Installation and Usage.md)
-* Developer Mail: <de...@kylin.incubator.apache.org>
+* Web Site: <http://kylin.incubator.apache.org/>
-* How To Contribute: See [wiki](https://github.com/KylinOLAP/Kylin/wiki/How-to-Contribute)
+* Developer Mail: <de...@kylin.incubator.apache.org>
-* Presentation: [Kylin Hadoop OLAP Engine v1.0](https://github.com/KylinOLAP/Kylin/blob/master/docs/Kylin_Hadoop_OLAP_Engine_v1.0.pdf?raw=true)
+* How To Contribute: See [this](docs/MISC/How to Contribute.md)
* Apache Proposal: [Apache Kylin](https://wiki.apache.org/incubator/KylinProposal)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4bd54078/docs/Installation/Frequently Asked Questions on Installation.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Frequently Asked Questions on Installation.md b/docs/Installation/Frequently Asked Questions on Installation.md
deleted file mode 100644
index c160403..0000000
--- a/docs/Installation/Frequently Asked Questions on Installation.md
+++ /dev/null
@@ -1,26 +0,0 @@
-Frequently Asked Questions on Installation
----
-* Some NPM error causes ERROR exit (中国大陆地区用户请特别注意此问题)?
-> Check out https://github.com/KylinOLAP/Kylin/issues/35
-
-* Can't get master address from ZooKeeper" when installing Kylin on Hortonworks Sandbox
-> Check out https://github.com/KylinOLAP/Kylin/issues/9.
-
-* Install scripted finished in my virtual machine, but cannot visit via http://localhost:9080
-> Check out https://github.com/KylinOLAP/Kylin/issues/12.
-
-* Map Reduce Job information can't display on sandbox deployment
-> Check out https://github.com/KylinOLAP/Kylin/issues/40
-
-* Install Kylin on CDH 5.2 or Hadoop 2.5.x
-> Check out discussion: https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/kylin-olap/X0GZfsX1jLc/nzs6xAhNpLkJ
->
-```
-I was able to deploy Kylin with following option in POM.
-<hadoop2.version>2.5.0</hadoop2.version>
-<yarn.version>2.5.0</yarn.version>
-<hbase-hadoop2.version>0.98.6-hadoop2</hbase-hadoop2.version>
-<zookeeper.version>3.4.5</zookeeper.version>
-<hive.version>0.13.1</hive.version>
-My Cluster is running on Cloudera Distribution CDH 5.2.0.
-```
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4bd54078/docs/MISC/FAQ on Kylin Installation and Usage.md
----------------------------------------------------------------------
diff --git a/docs/MISC/FAQ on Kylin Installation and Usage.md b/docs/MISC/FAQ on Kylin Installation and Usage.md
new file mode 100644
index 0000000..1c1291e
--- /dev/null
+++ b/docs/MISC/FAQ on Kylin Installation and Usage.md
@@ -0,0 +1,40 @@
+FAQ on Kylin Installation and Usage
+---
+#### Some NPM error causes ERROR exit (中国大陆地区用户请特别注意此问题)?
+Check out https://github.com/KylinOLAP/Kylin/issues/35
+
+#### Can't get master address from ZooKeeper" when installing Kylin on Hortonworks Sandbox
+Check out https://github.com/KylinOLAP/Kylin/issues/9.
+
+#### Install scripted finished in my virtual machine, but cannot visit via http://localhost:9080
+Check out https://github.com/KylinOLAP/Kylin/issues/12.
+
+#### Map Reduce Job information can't display on sandbox deployment
+Check out https://github.com/KylinOLAP/Kylin/issues/40
+
+#### Install Kylin on CDH 5.2 or Hadoop 2.5.x
+Check out discussion: https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/kylin-olap/X0GZfsX1jLc/nzs6xAhNpLkJ
+```
+I was able to deploy Kylin with following option in POM.
+<hadoop2.version>2.5.0</hadoop2.version>
+<yarn.version>2.5.0</yarn.version>
+<hbase-hadoop2.version>0.98.6-hadoop2</hbase-hadoop2.version>
+<zookeeper.version>3.4.5</zookeeper.version>
+<hive.version>0.13.1</hive.version>
+My Cluster is running on Cloudera Distribution CDH 5.2.0.
+```
+
+#### Unable to load a big cube as HTable, with java.lang.OutOfMemoryError: unable to create new native thread
+HBase (as of writing) allocates one thread per region when bulk loading a HTable. Try reduce the number of regions of your cube by setting its "capacity" to "MEDIUM" or "LARGE". Also tweaks OS & JVM can allow more threads, for example see [this article](http://blog.egilh.com/2006/06/2811aspx.html).
+
+#### Failed to run BuildCubeWithEngineTest, saying failed to connect to hbase while hbase is active
+User may get this error when first time run hbase client, please check the error trace to see whether there is an error saying couldn't access a folder like "/hadoop/hbase/local/jars"; If that folder doesn't exist, create it.
+
+#### SUM(field) returns a negtive result while all the numbers in this field are > 0
+If a column is declared as integer in Hive, the SQL engine (calcite) will use column's type (integer) as the data type for "SUM(field)", while the aggregated value on this field may exceed the scope of integer; in that case the cast will cause a negtive value be returned; The workround is, alter that column's type to BIGINT in hive, and then sync the table schema to Kylin (the cube doesn't need rebuild); Keep in mind that, always declare as BIGINT in hive for an integer column which would be used as a measure in Kylin; See hive number types: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-NumericTypes
+
+#### Why Kylin need extract the distinct columns from Fact Table before building cube?
+Kylin uses dictionary to encode the values in each column, this greatly reduce the cube's storage size. To build the dictionary, Kylin need fetch the distinct values for each column.
+
+#### Why Kylin calculate the HIVE table cardinality?
+The cardinality of dimensions is an important measure of cube complexity. The higher the cardinality, the bigger the cube, and thus the longer to build and the slower to query. Cardinality > 1,000 is worth attention and > 1,000,000 should be avoided at best effort. For optimal cube performance, try reduce high cardinality by categorize values or derive features.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4bd54078/docs/MISC/How to Contribute.md
----------------------------------------------------------------------
diff --git a/docs/MISC/How to Contribute.md b/docs/MISC/How to Contribute.md
new file mode 100644
index 0000000..e270606
--- /dev/null
+++ b/docs/MISC/How to Contribute.md
@@ -0,0 +1,42 @@
+#### Setup Dev Env
+* subscribe our developers' mail list via <de...@kylin.incubator.apache.org>
+* Fork on [GitHub](https://github.com/KylinOLAP)
+* ...
+
+#### Making Changes
+* Raise an issue on GitHub, describe the feature/enhancement/bug
+* Discuss with others in google group or issue comments, make sure the proposed changes fit in with what others are doing and have planned for the project
+* Make changes in your fork
+* Write unit test if no existing cover your change
+* Push to GitHub under your fork
+
+
+#### Contribute The Work
+* Raise a pull request on GitHub, include both **code** and **test**, link with **related issue**
+* Committer will review in terms of correctness, performance, design, coding style, test coverage
+* Discuss and revise if necessary
+* Finally committer merge code into main branch
+
+
+#### Wish List
+Some potential work items
+* Query Engine
+ * Cache generated class, reduce delay into ms level and avoid full GC triggered by perm generation
+ * [Issue #14](https://github.com/KylinOLAP/Kylin/issues/14) Derive meaningful cost in OLAP relational operator
+ * [Issue #15](https://github.com/KylinOLAP/Kylin/issues/15) Implement multi-column distinct count
+* Metadata
+ * [Issue #7](https://github.com/KylinOLAP/Kylin/issues/7) Merge multiple hbase tables
+* Job Engine
+ * [Issue #16](https://github.com/KylinOLAP/Kylin/issues/16) Tune HDFS & HBase parameters
+ * [Issue #17](https://github.com/KylinOLAP/Kylin/issues/17) Increase HDFS block size 1GB or close
+ * Shell command to support kill operation
+ * Use DoubleDouble instead of BigDecimal during cube build
+ * Drop quartz dependency, assess the cost/benefit first
+ * Cardinality run as one step job, allows progress tracking
+* ODBC/JDBC
+ * Test Kylin remote JDBC with java report tools
+ * Implement ODBC async mode, fetching from Kylin and feeding to client in parallel
+* Benchmark
+ * [Issue #18](https://github.com/KylinOLAP/Kylin/issues/18) Benchmark on standard dataset
+
+
[05/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2fce2ba7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2fce2ba7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2fce2ba7
Branch: refs/heads/streaming-localdict
Commit: 2fce2ba76d430cf3892373ae82b456b8c254d40c
Parents: bd9fc31
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 16:08:34 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 16:08:34 2015 +0800
----------------------------------------------------------------------
.../invertedindex/index/TableRecordInfo.java | 26 ----------------
.../invertedindex/model/IIKeyValueCodec.java | 32 +++++++++++++++++++-
2 files changed, 31 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2fce2ba7/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 7d87f99..3136ebb 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -108,33 +108,7 @@ public class TableRecordInfo {
return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
}
- public static TableRecordInfoDigest createDigest(int nColumns, boolean[] isMetric, String[] dataTypes, Map<Integer, Dictionary<?>> dictionaryMap) {
- int[] dictMaxIds = new int[nColumns];
- int[] lengths = new int[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- if (isMetric[i]) {
- final FixedLenMeasureCodec<?> fixedLenMeasureCodec = FixedLenMeasureCodec.get(DataType.getInstance(dataTypes[i]));
- lengths[i] = fixedLenMeasureCodec.getLength();
- } else {
- final Dictionary<?> dictionary = dictionaryMap.get(i);
- if (dictionary != null) {
- lengths[i] = dictionary.getSizeOfId();
- dictMaxIds[i] = dictionary.getMaxId();
- }
- }
- }
- // offsets
- int pos = 0;
- int[] offsets = new int[nColumns];
- for (int i = 0; i < nColumns; i++) {
- offsets[i] = pos;
- pos += lengths[i];
- }
-
- int byteFormLen = pos;
- return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
- }
public TableRecord createTableRecord() {
return new TableRecord(digest.createTableRecordBytes(), this);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2fce2ba7/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index d15f56b..ee26181 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -26,6 +26,8 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.index.*;
+import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.metadata.model.DataType;
import java.io.*;
import java.util.*;
@@ -122,6 +124,34 @@ public class IIKeyValueCodec implements KeyValueCodec {
// return new Decoder(kvs, incompleteDigest);
}
+ private static TableRecordInfoDigest createDigest(int nColumns, boolean[] isMetric, String[] dataTypes, Map<Integer, Dictionary<?>> dictionaryMap) {
+ int[] dictMaxIds = new int[nColumns];
+ int[] lengths = new int[nColumns];
+ for (int i = 0; i < nColumns; ++i) {
+ if (isMetric[i]) {
+ final FixedLenMeasureCodec<?> fixedLenMeasureCodec = FixedLenMeasureCodec.get(DataType.getInstance(dataTypes[i]));
+ lengths[i] = fixedLenMeasureCodec.getLength();
+ } else {
+ final Dictionary<?> dictionary = dictionaryMap.get(i);
+ if (dictionary != null) {
+ lengths[i] = dictionary.getSizeOfId();
+ dictMaxIds[i] = dictionary.getMaxId();
+ }
+ }
+ }
+ // offsets
+ int pos = 0;
+ int[] offsets = new int[nColumns];
+ for (int i = 0; i < nColumns; i++) {
+ offsets[i] = pos;
+ pos += lengths[i];
+ }
+
+ int byteFormLen = pos;
+
+ return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
+ }
+
private static class IIRowDecoder implements Iterable<Slice> {
private final TableRecordInfoDigest incompleteDigest;
@@ -184,7 +214,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
}
Preconditions.checkArgument(columns == incompleteDigest.getColumnCount(), "column count is " + columns + " should be equals to incompleteDigest.getColumnCount() " + incompleteDigest.getColumnCount());
- TableRecordInfoDigest digest = TableRecordInfo.createDigest(columns, incompleteDigest.getIsMetric(), incompleteDigest.getMetricDataTypes(), localDictionaries);
+ TableRecordInfoDigest digest = createDigest(columns, incompleteDigest.getIsMetric(), incompleteDigest.getMetricDataTypes(), localDictionaries);
Slice slice = new Slice(digest, curShard, curTimestamp, valueContainers);
slice.setLocalDictionaries(localDictionaries);
return slice;
[26/50] [abbrv] incubator-kylin git commit: Merge remote-tracking
branch 'origin/staging' into streaming-localdict
Posted by li...@apache.org.
Merge remote-tracking branch 'origin/staging' into streaming-localdict
Conflicts:
bin/kylin.sh
docs/Installation/install_and_run.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/696a6736
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/696a6736
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/696a6736
Branch: refs/heads/streaming-localdict
Commit: 696a6736698083b59bd12905da57e7c284696301
Parents: 84ec8e0 f22582d
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 20 10:21:03 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 20 10:21:03 2015 +0800
----------------------------------------------------------------------
README.md | 71 ++-----------
bin/kylin.sh | 18 ++--
bin/setenv.sh | 36 +++++++
...equently Asked Questions on Installation.pdf | Bin 52261 -> 0 bytes
docs/Installation/Hadoop Environment.md | 21 ++++
.../How to generate install binaries.md | 6 ++
docs/Installation/Installation.md | 38 +++++++
.../Installation/Multiple Kylin REST servers.md | 9 ++
...f Hadoop CLI Installation (Dev Env Setup).md | 76 -------------
... Hadoop CLI Installation (Dev Env Setup).pdf | Bin 44909 -> 0 bytes
.../Installation/Off Hadoop CLI Installation.md | 67 ++++++++++++
docs/Installation/On Hadoop CLI installation.md | 106 +++----------------
.../Installation/On Hadoop CLI installation.pdf | Bin 704398 -> 0 bytes
...On Hadoop Kylin installation using Docker.md | 2 +-
...n Hadoop Kylin installation using Docker.pdf | Bin 40098 -> 0 bytes
docs/Installation/off_cli_install_scene.png | Bin 0 -> 72567 bytes
docs/Installation/on_cli_install_scene.png | Bin 0 -> 53722 bytes
docs/README.md | 0
...in Cube Build and Job Monitoring Tutorial.md | 10 +-
...n Cube Build and Job Monitoring Tutorial.pdf | Bin 1054988 -> 0 bytes
docs/Tutorial/Kylin Cube Creation Tutorial.md | 9 +-
docs/Tutorial/Kylin Cube Creation Tutorial.pdf | Bin 1488946 -> 0 bytes
.../Kylin Cube Permission Grant Tutorial.pdf | Bin 358047 -> 0 bytes
docs/Tutorial/Kylin ODBC Driver Tutorial.md | 6 ++
docs/Tutorial/Kylin ODBC Driver Tutorial.pdf | Bin 34429 -> 0 bytes
docs/Tutorial/Kylin Web Tutorial.md | 8 +-
docs/Tutorial/Kylin Web Tutorial.pdf | Bin 1714951 -> 0 bytes
docs/Tutorial/Kylin and Excel Tutorial.md | 3 +
docs/Tutorial/Kylin and Tableau Tutorial.md | 4 +-
docs/Tutorial/Kylin and Tableau Tutorial.pdf | Bin 1246553 -> 0 bytes
docs/Tutorial/Quick play with a sample cube.md | 20 ++++
.../job/hadoop/cube/BaseCuboidJobTest.java | 3 +
.../job/hadoop/cube/MergeCuboidJobTest.java | 4 +-
.../kylin/job/hadoop/cube/NDCuboidJobTest.java | 4 +-
.../cube/RangeKeyDistributionJobTest.java | 4 +-
old_README.md | 94 ++++++++++++++++
pom.xml | 4 +
query/src/test/resources/query/sql/query78.sql | 4 +
.../kylin/storage/hbase/CubeStorageEngine.java | 15 ++-
39 files changed, 387 insertions(+), 255 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/696a6736/bin/kylin.sh
----------------------------------------------------------------------
diff --cc bin/kylin.sh
index 7d85368,232feef..179fa47
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@@ -37,9 -34,9 +36,10 @@@ the
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
- export JAVA_OPTS="-Xms2048M -Xmx2048M"
++ export JAVA_OPTS="-Xms2048M -Xmx2048M"
- hbase ${JAVA_OPTS} -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
+ hbase ${KYLIN_EXTRA_START_OPTS} \
+ -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/696a6736/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/696a6736/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
[50/50] [abbrv] incubator-kylin git commit: Merge branch
'origin/staging'
Posted by li...@apache.org.
Merge branch 'origin/staging'
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cc1fed44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cc1fed44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cc1fed44
Branch: refs/heads/streaming-localdict
Commit: cc1fed44d5ac5456ec3342ced66bbe414c631c2d
Parents: ee2bb66 3655b33
Author: honma <ho...@ebay.com>
Authored: Fri Mar 20 09:50:26 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Fri Mar 20 09:50:26 2015 -0700
----------------------------------------------------------------------
README.md | 51 +-
...requently Asked Questions on Installation.md | 26 -
docs/JobEngine/Design.md | 2 +-
.../MISC/FAQ on Kylin Installation and Usage.md | 40 +
docs/MISC/How to Contribute.md | 42 +
docs/MISC/Kylin Document Conventions.md | 9 +
.../Advanced settings of Kylin environment.md | 28 +
docs/Operations/Export Kylin HBase data.md | 19 +
docs/Operations/Kylin Metadata Store.md | 31 +
docs/REST/Build Cube with Restful API.md | 51 +
...w to consume Kylin REST API in javascript.md | 42 +
docs/REST/Kylin Restful API List.md | 2020 ++++++++++++++++++
.../TechInside/New meta data model structure.md | 18 +
...n Kylin test case with HBase Mini Cluster.md | 15 +
.../How to use kylin remote jdbc driver.md | 83 +
docs/Tutorial/Kylin and Tableau Tutorial.md | 2 +-
old_README.md | 94 -
query/src/test/resources/query/sql/query78.sql | 2 +-
18 files changed, 2446 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
[29/50] [abbrv] incubator-kylin git commit: KYLIN-650 add operations
sections
Posted by li...@apache.org.
KYLIN-650 add operations sections
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/284ef260
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/284ef260
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/284ef260
Branch: refs/heads/streaming-localdict
Commit: 284ef260140d0cae845dfb6075a1e679f9e9300c
Parents: 4bd5407
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 23:07:09 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 23:07:09 2015 -0700
----------------------------------------------------------------------
README.md | 14 ++++++++-
.../Advanced settings of Kylin environment.md | 28 ++++++++++++++++++
docs/Operations/Export Kylin HBase data.md | 19 ++++++++++++
docs/Operations/Kylin Metadata Store.md | 31 ++++++++++++++++++++
4 files changed, 91 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/284ef260/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 2489576..fa180b0 100644
--- a/README.md
+++ b/README.md
@@ -17,8 +17,20 @@ Tutorial
Please follow this installation tutorial to start with Kylin: [Installation Tutorial](docs/Installation/Installation.md)
-Advance Usage
+Advance Topics
-------
+####REST
+
+####Operations
+1.[Kylin Metadata Store](docs/Operations/Kylin Metadata Store.md)
+
+2.[Export Kylin HBase data](docs/Operations/Export Kylin HBase data.md)
+
+3.[Advanced settings of Kylin environment](docs/Operations/Advanced settings of Kylin environment.md)
+
+####Test
+
+####Technial Details
Get Help
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/284ef260/docs/Operations/Advanced settings of Kylin environment.md
----------------------------------------------------------------------
diff --git a/docs/Operations/Advanced settings of Kylin environment.md b/docs/Operations/Advanced settings of Kylin environment.md
new file mode 100644
index 0000000..eabfa43
--- /dev/null
+++ b/docs/Operations/Advanced settings of Kylin environment.md
@@ -0,0 +1,28 @@
+Advanced settings of Kylin environment
+----
+
+### A. Enabling LZO compression
+
+LZO compression can be leveraged to compress the output of MR jobs, as well as hbase table storage, reducing the storage overhead. By default we do not enable LZO compression in Kylin because hadoop sandbox venders tend to not include LZO in their distributions due to license(GPL) issues.
+
+To enable LZO in Kylin, there are three steps:
+
+#### Make sure LZO is working in your environment
+
+we have a simple tool to test whether LZO is well installed and configured in your environment(we only test it on the hadoop CLI that you deployed Kylin), Just run
+
+`hbase org.apache.hadoop.util.RunJar kylin-job-latest.jar com.kylinolap.job.tools.LZOSupportnessChecker`
+
+If the program prints "LZO supported by current env? true", you're good to go. Otherwise you'll need to first install LZO properly.
+
+#### Modify kylin_job_conf.xml
+
+You'll need to stop Kylin first by running `${KYLIN_HOME}/bin/kylin.sh stop`, and then modify /etc ${KYLIN_HOME}/conf/kylin_job_conf.xml by uncommenting some configuration entries related to LZO compression.
+
+#### export KYLIN_LD_LIBRARY_PATH to carry your native library paths
+
+Before starting Kylin again, you'll need to uncomment KYLIN_LD_LIBRARY_PATH in ${KYLIN_HOME}/bin/setenv.sh to carry your LZO native library paths. Here's an example for hdp 2.1:
+
+ export KYLIN_LD_LIBRARY_PATH=::/usr/lib/hadoop/lib/native/Linux-amd64-64:/usr/lib/hadoop/lib/native
+
+After exporting, you need to run `${KYLIN_HOME}/bin/kylin.sh start` to start Kylin again. Now Kylin will use LZO to compress MR outputs and hbase tables.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/284ef260/docs/Operations/Export Kylin HBase data.md
----------------------------------------------------------------------
diff --git a/docs/Operations/Export Kylin HBase data.md b/docs/Operations/Export Kylin HBase data.md
new file mode 100644
index 0000000..90c4778
--- /dev/null
+++ b/docs/Operations/Export Kylin HBase data.md
@@ -0,0 +1,19 @@
+# Export Kylin HBase data
+
+Kylin persists all data (meta data and cube) in HBase; You may want to export the data sometimes for whatever purposes (backup, migration, troubleshotting etc); This page describes the steps to do this and also there is a Java app for you to do this easily;
+
+Steps:
+
+1. Cleanup unused cubes to save storage space (be cautious on production!): run the following command in hbase CLI:
+```
+hbase org.apache.hadoop.util.RunJar /${KYLIN_HOME}/lib/kylin-job-(version).jar org.apache.kylin.job.hadoop.cube.StorageCleanupJob --delete true
+```
+2. List all HBase tables, iterate and then export each Kylin table to HDFS; See https://hbase.apache.org/book/ops_mgt.html#export
+
+3. Copy the export folder from HDFS to local file system, and then archive it;
+
+4. (optional) Download the archive from Hadoop CLI to local;
+
+5. Cleanup the export folder from CLI HDFS and local file system;
+
+Kylin provide the "ExportHBaseData.java" (currently only exist in "minicluster" branch) for you to do the step 2-5 in one run; Please ensure the correct path of "kylin.properties" has been set in the sys env; This Java uses the sandbox config by default;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/284ef260/docs/Operations/Kylin Metadata Store.md
----------------------------------------------------------------------
diff --git a/docs/Operations/Kylin Metadata Store.md b/docs/Operations/Kylin Metadata Store.md
new file mode 100644
index 0000000..33c1afe
--- /dev/null
+++ b/docs/Operations/Kylin Metadata Store.md
@@ -0,0 +1,31 @@
+Kylin organizes all of its metadata(including cube descriptions and instances, projects, inverted index description and instances, jobs, tables and dictionaries) as a hierarchy file system. However, Kylin uses hbase to store it, rather than normal file system. If you check your kylin configuration file(kylin.properties) you will find such a line:
+
+`# The metadata store in hbase`
+`kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-unsecure`
+
+This indicates that the metadata will be saved as a htable called `kylin_metadata_qa`. You can scan the htable in hbase shell to check it out.
+
+# Backup Metadata Store
+
+Sometimes you need to backup the Kylin's Metadata Store from hbase to your disk file system.
+In such cases, assuming you're on the hadoop CLI(or sandbox) where you deployed Kylin, you can use:
+
+ mkdir ~/meta_dump
+
+ hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar com.kylinolap.common.persistence.ResourceTool copy ${KYLIN_HOME}/conf/kylin.properties ~/meta_dump
+
+to dump your metadata to your local folder ~/meta_dump.
+
+# Restore Metadata Store
+
+In case you find your metadata store messed up, and you want to restore to a previous backup:
+
+first clean up the metadata store:
+
+ hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar com.kylinolap.common.persistence.ResourceTool reset
+
+then upload the backup metadata in ~/meta_dump to Kylin's metadata store:
+
+ hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar com.kylinolap.common.persistence.ResourceTool copy ~/meta_dump ${KYLIN_HOME}/conf/kylin.properties
+
+
[22/50] [abbrv] incubator-kylin git commit: KYLIN-650 add "what's
next" part
Posted by li...@apache.org.
KYLIN-650 add "what's next" part
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ae75f5f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ae75f5f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ae75f5f4
Branch: refs/heads/streaming-localdict
Commit: ae75f5f408c123b9b140f0334e185cd475159724
Parents: da16316
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 19:10:03 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 19:10:03 2015 -0700
----------------------------------------------------------------------
.../Kylin Cube Build and Job Monitoring Tutorial.md | 10 +++++++++-
docs/Tutorial/Kylin ODBC Driver Tutorial.md | 1 -
docs/Tutorial/Quick play with a sample cube.md | 8 +++++---
3 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae75f5f4/docs/Tutorial/Kylin Cube Build and Job Monitoring Tutorial.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin Cube Build and Job Monitoring Tutorial.md b/docs/Tutorial/Kylin Cube Build and Job Monitoring Tutorial.md
index 827cd94..452f24b 100644
--- a/docs/Tutorial/Kylin Cube Build and Job Monitoring Tutorial.md
+++ b/docs/Tutorial/Kylin Cube Build and Job Monitoring Tutorial.md
@@ -57,4 +57,12 @@ Click the icon button show in each step to see the details: `Parameters`, `Log`,
- 
\ No newline at end of file
+ 
+
+#### What's next
+
+After cube being built, you might need to:
+
+1. [Query the cube via web interface](Kylin Web Tutorial.md)
+2. [Query the cube via ODBC](Kylin ODBC Driver Tutorial.md)
+3. [Grant permission to cubes](Kylin Cube Permission Grant Tutorial.md)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae75f5f4/docs/Tutorial/Kylin ODBC Driver Tutorial.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin ODBC Driver Tutorial.md b/docs/Tutorial/Kylin ODBC Driver Tutorial.md
index 7f84213..17653fd 100644
--- a/docs/Tutorial/Kylin ODBC Driver Tutorial.md
+++ b/docs/Tutorial/Kylin ODBC Driver Tutorial.md
@@ -32,6 +32,5 @@ Kylin ODBC Driver Tutorial
2. [Kylin and Excel Tutorial](Kylin and Excel Tutorial.md)
-
## Bug Report
Open github issue here
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae75f5f4/docs/Tutorial/Quick play with a sample cube.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Quick play with a sample cube.md b/docs/Tutorial/Quick play with a sample cube.md
index 37c44ff..9495b4a 100644
--- a/docs/Tutorial/Quick play with a sample cube.md
+++ b/docs/Tutorial/Quick play with a sample cube.md
@@ -10,9 +10,11 @@ Kylin provides a script for you to create a sample Cube; the script will also cr
select cal_dt, sum(price) as total_selled, count(distinct seller_id) as sellers from kylin_sales group by cal_dt order by cal_dt
6. You can verify the query result and compare the response time with hive;
+
#### What's next
-After cube being created, you might need to:
+After cube being built, you might need to:
-1. [Build the cube so that it can be queried](Kylin Cube Build and Job Monitoring Tutorial.md)
-2. [Grant permission to cubes](Kylin Cube Permission Grant Tutorial.md)
\ No newline at end of file
+1. [Query the cube via web interface](Kylin Web Tutorial.md)
+2. [Query the cube via ODBC](Kylin ODBC Driver Tutorial.md)
+3. [Grant permission to cubes](Kylin Cube Permission Grant Tutorial.md)
\ No newline at end of file
[04/50] [abbrv] incubator-kylin git commit: fix
Posted by li...@apache.org.
fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bd9fc315
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bd9fc315
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bd9fc315
Branch: refs/heads/streaming-localdict
Commit: bd9fc315bdb8f41f907f0bef07109ca16b0e41b1
Parents: 303f8ff
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 16:00:10 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 16:00:10 2015 +0800
----------------------------------------------------------------------
.../invertedindex/index/TableRecordInfo.java | 24 +++++++++++---------
1 file changed, 13 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bd9fc315/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index a3af176..7d87f99 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -18,10 +18,12 @@
package org.apache.kylin.invertedindex.index;
+import com.google.common.collect.Maps;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
@@ -72,19 +74,19 @@ public class TableRecordInfo {
isMetric[i] = desc.isMetricsCol(i);
dataTypes[i] = tblColRef.getDatatype();
if (isMetric[i]) {
- final DataType dataType = DataType.getInstance(tblColRef.getColumn().getDatatype());
- if (dataType.isNumberFamily()) {
- lengths[i] = 16;
- } else if (dataType.isStringFamily()){
- lengths[i] = 256;
- } else if (dataType.isDateTimeFamily()) {
- lengths[i] = 10;
- } else {
- throw new RuntimeException("invalid data type:" + dataType);
- }
+ lengths[i] = FixedLenMeasureCodec.get(DataType.getInstance(tblColRef.getColumn().getDatatype())).getLength();
} else {
if (dictionaryMap.isEmpty()) {
- lengths[i] = FixedLenMeasureCodec.get(tblColRef.getColumn().getType()).getLength();
+ final DataType dataType = DataType.getInstance(tblColRef.getColumn().getDatatype());
+ if (dataType.isNumberFamily()) {
+ lengths[i] = 16;
+ } else if (dataType.isStringFamily()){
+ lengths[i] = 256;
+ } else if (dataType.isDateTimeFamily()) {
+ lengths[i] = 10;
+ } else {
+ throw new RuntimeException("invalid data type:" + dataType);
+ }
dictMaxIds[i] = Integer.MAX_VALUE;
} else {
final Dictionary<?> dictionary = dictionaryMap.get(i);
[32/50] [abbrv] incubator-kylin git commit: KYLIN-650 cleaning
Posted by li...@apache.org.
KYLIN-650 cleaning
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/af42057a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/af42057a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/af42057a
Branch: refs/heads/streaming-localdict
Commit: af42057ad2ff01a7ffca32b69091177af9ee80fa
Parents: 5acdcd2
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 23:27:00 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 23:27:00 2015 -0700
----------------------------------------------------------------------
README.md | 2 +
docs/JobEngine/Design.md | 2 +-
old_README.md | 94 -------------------------------------------
3 files changed, 3 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af42057a/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index f3d383f..fba7c0a 100644
--- a/README.md
+++ b/README.md
@@ -48,6 +48,8 @@ Advance Topics
1.[New meta data model structure](docs/TechInside/New meta data model structure.md)
+2.[Job Engine Design](docs/JobEngine/Design.md)
+
Get Help
------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af42057a/docs/JobEngine/Design.md
----------------------------------------------------------------------
diff --git a/docs/JobEngine/Design.md b/docs/JobEngine/Design.md
index 9c1a94d..59b9ebe 100644
--- a/docs/JobEngine/Design.md
+++ b/docs/JobEngine/Design.md
@@ -4,7 +4,7 @@
### Executable
-
+
**Executable** is a top-level interface for all kinds of jobs or tasks.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af42057a/old_README.md
----------------------------------------------------------------------
diff --git a/old_README.md b/old_README.md
deleted file mode 100644
index 44f6859..0000000
--- a/old_README.md
+++ /dev/null
@@ -1,94 +0,0 @@
-Kylin OLAP
-===========
-
-
-Overview
-------------
-> Extreme OLAP Engine for Big Data
-
-An open source distributed Analytical Engine from eBay to provide SQL interface and multi-dimensional analysis (OLAP) on Hadoop to support TB to PB size analysis.
-
-
-Getting Started
----------------
-
-### Prerequisites ###
-
-#### Hadoop ####
-- Hadoop: 2.2.0.2.0.6.0-61 or above
-- Hive: 0.12.0.2.0.6.0-61 or above
-- HBase: 0.96.0.2.0.6.0-61-hadoop2
-
-
-_Tested with Hortonworks HDP 2.1.3 and Cloudera Quickstart VM 5.1._
-
-#### Misc ####
-- Command hadoop, hive, hbase is workable on your hadoop cluster
-- JDK Runtime: JDK7 (OpenJDK or Oracle JDK)
-- Maven
-- Git
-- Tomcat (CATALINA_HOME being set)
-- Npm
-
-#### Before Install ####
-- For the quick tutorial, we assume that your hadoop has priviledges disabled which means any user could run hadoop and hive command
-
-
-Installation
-------------
-It is very easy to install Kylin for exploration/development. There are 3 supported options:
-
-
-1. Sandbox (HDP or CDH sandbox)
-2. Dev Environment (IDE + Sandbox)
-3. Docker container (Docker)
-
-#### Sandbox (On-Hadoop-CLI installation) ####
-
-If you are free to install Kylin on your hadoop CLI machine or Hadoop sandbox, this is the most convenient scenario, for it puts everything in a single machine.
-
-
-
-
-
-For normal users, the application in the above picture means `Kylin Web`, which contains a web interface for cube building, querying and all sorts of management. Kylin Web launches a query engine for querying and a cube build engine for building cubes. These two engines interact with the components in Hadoop CLI, like hive and hbase.
-
-For a hands-on tutorial please visit [On-Hadoop-CLI installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-CLI-installation).
-
-#### Dev Environment (Off-Hadoop-CLI Installation) ####
-
-This is typically for development environment setup.
-
-Applications does not necessarily mean Kylin Web, it can also be any test cases or java applications running at your local IDE(query engine and cube build engine can be launched programmatically in your code). In this case, it is no longer reasonable to assume that all the Kylin components reside in the same machine as your Hadoop CLI. Fortunately, Kylin still works under such condition with proper settings on your CLI machine.
-
-
-
-For a hands-on tutorial please visit [Off-Hadoop-CLI installation](https://github.com/KylinOLAP/Kylin/wiki/Off-Hadoop-CLI-Installation-(Dev-Env-Setup))
-
-#### Docker Container ####
-With help from [SequenceIQ](http://sequenceiq.com/), there's docker container for Kylin (along with Hadoop, HBase and Hive) available now:[sequenceiq/docker-kylin](https://github.com/sequenceiq/docker-kylin). The only thing you will need to do is to pull the container from the official Docker repository to be up and running in few minutes.
-
-Features:
-
-- Size - Light weight compared to downloading and setting up HDP or CDH sandbox.
-- Cluster support - Supports multi node installation.
-- Fully Automated - No manual steps. One command does it all
-
-For a hands-on tutorial please visit [Kylin Docker installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-Kylin-installation-using-Docker).
-
-
-
-### Resources ###
-
-* Web Site: <http://kylin.io>
-
-* Google Group: [Kylin OLAP Group](https://groups.google.com/forum/#!forum/kylin-olap)
-
-* Developer Mail: <ky...@googlegroups.com>
-
-* How To Contribute: See [wiki](https://github.com/KylinOLAP/Kylin/wiki/How-to-Contribute)
-
-* Presentation: [Kylin Hadoop OLAP Engine v1.0](https://github.com/KylinOLAP/Kylin/blob/master/docs/Kylin_Hadoop_OLAP_Engine_v1.0.pdf?raw=true)
-
-* Apache Proposal: [Apache Kylin](https://wiki.apache.org/incubator/KylinProposal)
-
[17/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c952a36f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c952a36f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c952a36f
Branch: refs/heads/streaming-localdict
Commit: c952a36fc8292e72d0dec81d1fcc0b5de3349afa
Parents: dcc7217 73ad4d4
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Thu Mar 19 18:14:00 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Thu Mar 19 18:14:00 2015 +0800
----------------------------------------------------------------------
.../storage/gridtable/GTAggregateScanner.java | 18 +++++++++---------
.../apache/kylin/storage/gridtable/GTInfo.java | 2 +-
.../kylin/storage/gridtable/GTRawScanner.java | 5 +++++
.../apache/kylin/storage/gridtable/GridTable.java | 12 ++++++++----
.../kylin/storage/gridtable/IGTScanner.java | 2 ++
5 files changed, 25 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[14/50] [abbrv] incubator-kylin git commit: fix export result bug
Posted by li...@apache.org.
fix export result bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cf99527b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cf99527b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cf99527b
Branch: refs/heads/streaming-localdict
Commit: cf99527bd5de7329e6526646921a29f51653a0a3
Parents: 0170668
Author: jiazhong <ji...@ebay.com>
Authored: Thu Mar 19 17:43:53 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Thu Mar 19 17:43:53 2015 +0800
----------------------------------------------------------------------
webapp/app/partials/query/query_detail.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cf99527b/webapp/app/partials/query/query_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/query/query_detail.html b/webapp/app/partials/query/query_detail.html
index a4c11bf..7157e9c 100644
--- a/webapp/app/partials/query/query_detail.html
+++ b/webapp/app/partials/query/query_detail.html
@@ -88,7 +88,7 @@
<span ng-if="curQuery.graph.show"><i class="fa fa-list-ul"></i> Grid</span>
</button>
<a class="btn btn-default btn-xs"
- href="{{config.service.url}}query/format/csv?sql={{sanitate(curQuery.sql)}}&project={{state.selectedProject}}" target="_blank"><i
+ href="{{config.service.url}}query/format/csv?sql={{sanitate(curQuery.sql)}}&project={{curQuery.project}}" target="_blank"><i
class="fa fa-cloud-download"></i> Export</a>
<a class="btn btn-default btn-xs"
ng-click="refreshUi();"><i
[03/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/303f8ffc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/303f8ffc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/303f8ffc
Branch: refs/heads/streaming-localdict
Commit: 303f8ffcd6ae1a348dc401a35f5be7c2cb065cc6
Parents: 5b3413b
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 15:49:45 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 15:49:45 2015 +0800
----------------------------------------------------------------------
.../kylin/invertedindex/index/TableRecordInfo.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/303f8ffc/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index d93ff77..a3af176 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -72,7 +72,16 @@ public class TableRecordInfo {
isMetric[i] = desc.isMetricsCol(i);
dataTypes[i] = tblColRef.getDatatype();
if (isMetric[i]) {
- lengths[i] = FixedLenMeasureCodec.get(DataType.getInstance(tblColRef.getColumn().getDatatype())).getLength();
+ final DataType dataType = DataType.getInstance(tblColRef.getColumn().getDatatype());
+ if (dataType.isNumberFamily()) {
+ lengths[i] = 16;
+ } else if (dataType.isStringFamily()){
+ lengths[i] = 256;
+ } else if (dataType.isDateTimeFamily()) {
+ lengths[i] = 10;
+ } else {
+ throw new RuntimeException("invalid data type:" + dataType);
+ }
} else {
if (dictionaryMap.isEmpty()) {
lengths[i] = FixedLenMeasureCodec.get(tblColRef.getColumn().getType()).getLength();
[19/50] [abbrv] incubator-kylin git commit: add a separate setenv.sh
for jvm settings, debug settings and native libarary
Posted by li...@apache.org.
add a separate setenv.sh for jvm settings, debug settings and native libarary
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a540f402
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a540f402
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a540f402
Branch: refs/heads/streaming-localdict
Commit: a540f4024134f2198c52c0a8dfede10da2a08f39
Parents: 014ef83
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 16:10:11 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 18:56:21 2015 -0700
----------------------------------------------------------------------
bin/kylin.sh | 16 ++++++++--------
bin/setenv.sh | 36 ++++++++++++++++++++++++++++++++++++
2 files changed, 44 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a540f402/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
index a6983df..232feef 100644
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -7,12 +7,6 @@ then
tomcat_root=${dir}/../tomcat
export tomcat_root
- #if [ ! -z "$KYLIN_LD_LIBRARY_PATH" ]
- #then
- # echo "KYLIN_LD_LIBRARY_PATH is set to $KYLIN_LD_LIBRARY_PATH"
- #else
- # exit 1
- #fi
#The location of all hadoop/hbase configurations are difficult to get.
#Plus, some of the system properties are secretly set in hadoop/hbase shell command.
@@ -23,7 +17,6 @@ then
#In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
#hbase command will do all the dirty tasks for us:
- #-Djava.library.path=${KYLIN_LD_LIBRARY_PATH} \
useSandbox=`cat ${KYLIN_HOME}/conf/kylin.properties | grep 'kylin.sandbox' | awk -F '=' '{print $2}'`
@@ -31,12 +24,19 @@ then
if [ "$useSandbox" = "true" ]
then spring_profile="sandbox"
fi
+
+ #retrive $hive_dependency
source ${dir}/find-hive-dependency.sh
+ #retrive $KYLIN_EXTRA_START_OPTS
+ if [ -f "${dir}/setenv.sh" ]
+ then source ${dir}/setenv.sh
+ fi
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
- hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
+ hbase ${KYLIN_EXTRA_START_OPTS} \
+ -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a540f402/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/setenv.sh b/bin/setenv.sh
new file mode 100755
index 0000000..53f614d
--- /dev/null
+++ b/bin/setenv.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# uncomment following to for it to take effect
+# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC"
+
+# uncomment following to for it to take effect
+# export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -javaagent:${KYLIN_HOME}/lib/CustomAgent.jar -Dcom.ebay.lnp.rmiagent.port=8080"
+
+# uncomment following to for it to take effect
+# export KYLIN_LD_LIBRARY_SETTINGS="-Djava.library.path=/apache/hadoop/lib/native/Linux-amd64-64"
+
+export KYLIN_EXTRA_START_OPTS=""
+
+if [ ! -z "${KYLIN_JVM_SETTINGS}" ]
+then
+ echo "KYLIN_JVM_SETTINGS is ${KYLIN_JVM_SETTINGS}"
+ KYLIN_EXTRA_START_OPTS="${KYLIN_JVM_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
+else
+ echo "KYLIN_JVM_SETTINGS is not set, using default jvm settings"
+fi
+
+if [ ! -z "${KYLIN_DEBUG_SETTINGS}" ]
+then
+ echo "KYLIN_DEBUG_SETTINGS is ${KYLIN_DEBUG_SETTINGS}"
+ KYLIN_EXTRA_START_OPTS="${KYLIN_DEBUG_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
+else
+ echo "KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging"
+fi
+
+if [ ! -z "${KYLIN_LD_LIBRARY_SETTINGS}" ]
+then
+ echo "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}"
+ KYLIN_EXTRA_START_OPTS="${KYLIN_LD_LIBRARY_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
+else
+ echo "KYLIN_LD_LIBRARY_SETTINGS is not set, lzo compression at MR and hbase might not work"
+fi
[46/50] [abbrv] incubator-kylin git commit: Build cube in mem
Posted by li...@apache.org.
Build cube in mem
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4306645b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4306645b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4306645b
Branch: refs/heads/streaming-localdict
Commit: 4306645bcadefde268461a6634f784a073e506dd
Parents: c952a36
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Mar 20 18:13:50 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Mar 20 18:13:50 2015 +0800
----------------------------------------------------------------------
.../metadata/measure/MeasureAggregator.java | 2 +-
.../apache/kylin/storage/gridtable/GTInfo.java | 2 +-
.../storage/gridtable/GTSampleCodeSystem.java | 2 +-
.../kylin/streaming/cube/CubeStreamBuilder.java | 95 +++++++++++++-------
4 files changed, 64 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4306645b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
index 634c85f..0e2fc55 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
@@ -59,7 +59,7 @@ abstract public class MeasureAggregator<V> {
}
public static boolean isBigDecimal(String type) {
- return "decimal".equalsIgnoreCase(type);
+ return type.startsWith("decimal");
}
public static boolean isDouble(String type) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4306645b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
index 04e720c..32c4cfc 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -157,7 +157,7 @@ public class GTInfo {
/** required */
public Builder setPrimaryKey(BitSet primaryKey) {
- info.primaryKey = primaryKey;
+ info.primaryKey = (BitSet)primaryKey.clone();
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4306645b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
index b7f12df..ae6a205 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
@@ -78,7 +78,7 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
@Override
public MeasureAggregator<?> newMetricsAggregator(String aggrFunction, int col) {
- return MeasureAggregator.create(aggrFunction, info.colTypes[col].getName());
+ return MeasureAggregator.create(aggrFunction, info.colTypes[col].toString());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4306645b/streaming/src/main/java/org/apache/kylin/streaming/cube/CubeStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/cube/CubeStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/cube/CubeStreamBuilder.java
index 0ae9a5e..3b81f44 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/cube/CubeStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/cube/CubeStreamBuilder.java
@@ -37,6 +37,7 @@ import com.google.common.base.Function;
import com.google.common.collect.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -51,8 +52,10 @@ import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.metadata.measure.MeasureAggregators;
import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.serializer.DataTypeSerializer;
import org.apache.kylin.storage.gridtable.*;
import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
import org.apache.kylin.streaming.Stream;
@@ -84,11 +87,11 @@ public class CubeStreamBuilder extends StreamBuilder {
private MeasureCodec measureCodec;
private MeasureAggregators aggs = null;
private int measureNumber;
-
+ public static final LongWritable ONE = new LongWritable(1l);
private String[] metricsAggrFuncs = null;
public CubeStreamBuilder(LinkedBlockingDeque<Stream> queue, String hTableName, CubeDesc desc, int partitionId) {
- super(queue, 2000);
+ super(queue, 10000);
this.desc = desc;
this.partitionId = partitionId;
this.cuboidScheduler = new CuboidScheduler(desc);
@@ -107,19 +110,17 @@ public class CubeStreamBuilder extends StreamBuilder {
metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
}
metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
+
+
}
@Override
protected void build(List<Stream> streamsToBuild) {
+ long startTime = System.currentTimeMillis();
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- try {
- cubeSegment = cubeManager.appendSegments(cube, System.currentTimeMillis());
- } catch (IOException e) {
- e.printStackTrace();
- }
- intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), cubeSegment);
+ intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), null);
table = Lists.transform(streamsToBuild, new Function<Stream, List<String>>() {
@Nullable
@@ -141,6 +142,7 @@ public class CubeStreamBuilder extends StreamBuilder {
}
assert result.size() > 0;
+ logger.info("Totally " + result.size() + " cuboids be calculated, takes " + (System.currentTimeMillis() - startTime)/1000.0 + " seconds");
}
@@ -161,17 +163,34 @@ public class CubeStreamBuilder extends StreamBuilder {
}
result.add(thisCuboid);
+ logger.info("Cuboid " + cuboidId + " is built.");
+ // outputGT(thisCuboid);
}
+ private void outputGT(GridTable gridTable) throws IOException {
+ IGTScanner scanner = gridTable.scan(null, null, null, null);
+ for (GTRecord record : scanner) {
+ logger.debug(record.toString());
+ }
+ }
private GridTable calculateBaseCuboid(List<List<String>> table, long baseCuboidId) throws IOException {
+ logger.info("Calculating base cuboid " + baseCuboidId + ", source records number " + table.size() );
+ Cuboid baseCuboid = Cuboid.findById(this.desc, baseCuboidId);
+ DataTypeSerializer[] serializers = new DataTypeSerializer[baseCuboid.getColumns().size()];
+
+ for (int i = 0; i < baseCuboid.getColumns().size(); i++) {
+ serializers[i] = DataTypeSerializer.create(baseCuboid.getColumns().get(i).getType());
+ }
+
+
GridTable gridTable = newGridTableByCuboidID(baseCuboidId);
GTRecord r = new GTRecord(gridTable.getInfo());
GTBuilder builder = gridTable.rebuild();
for (List<String> row : table) {
- String[] dimensions = buildKey(row);
+ Object[] dimensions = buildKey(row, serializers);
Object[] metricsValues = buildValue(row);
Object[] recordValues = new Object[dimensions.length + metricsValues.length];
System.arraycopy(dimensions, 0, recordValues, 0, dimensions.length);
@@ -183,7 +202,6 @@ public class CubeStreamBuilder extends StreamBuilder {
IGTScanner scanner = gridTable.scanAndAggregate(null, null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
-
GridTable baseCuboidGridTable = newGridTableByCuboidID(baseCuboidId);
builder = baseCuboidGridTable.rebuild();
for (GTRecord record : scanner) {
@@ -198,15 +216,17 @@ public class CubeStreamBuilder extends StreamBuilder {
GTInfo info = newGTInfo(cuboidID);
GTSimpleMemStore store = new GTSimpleMemStore(info);
GridTable gridTable = new GridTable(info, store);
-
return gridTable;
}
private GridTable aggregateCuboid(GridTable parentCuboid, long parentCuboidId, long cuboidId) throws IOException {
+ //logger.info("Calculating cuboid " + cuboidId + " from parent " + parentCuboidId);
Pair<BitSet, BitSet> columnBitSets = getDimensionAndMetricColumBitSet(parentCuboidId);
+ BitSet parentDimensions = columnBitSets.getFirst();
+ BitSet measureColumns = columnBitSets.getSecond();
+ BitSet childDimensions = (BitSet) parentDimensions.clone();
- BitSet dimensionBitSet = columnBitSets.getFirst();
long mask = Long.highestOneBit(parentCuboidId);
long childCuboidId = cuboidId;
long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(parentCuboidId);
@@ -215,19 +235,24 @@ public class CubeStreamBuilder extends StreamBuilder {
if ((mask & parentCuboidId) > 0) {
if ((mask & childCuboidId) == 0) {
// this dim will be aggregated
- dimensionBitSet.set(index, false);
+ childDimensions.set(index, false);
}
index++;
}
mask = mask >> 1;
}
-
- IGTScanner scanner = parentCuboid.scanAndAggregate(null, null, dimensionBitSet, columnBitSets.getSecond(), metricsAggrFuncs, null);
+ IGTScanner scanner = parentCuboid.scanAndAggregate(null, null, childDimensions, measureColumns, metricsAggrFuncs, null);
GridTable cuboidGridTable = newGridTableByCuboidID(cuboidId);
GTBuilder builder = cuboidGridTable.rebuild();
+ GTRecord r = new GTRecord(cuboidGridTable.getInfo());
+
+ BitSet allNeededColumns = new BitSet();
+ allNeededColumns.or(childDimensions);
+ allNeededColumns.or(measureColumns);
for (GTRecord record : scanner) {
- builder.write(record);
+ buildNewGTRecord(record, r, allNeededColumns);
+ builder.write(r);
}
builder.close();
@@ -235,6 +260,12 @@ public class CubeStreamBuilder extends StreamBuilder {
return cuboidGridTable;
}
+ private void buildNewGTRecord(GTRecord record, GTRecord newRecord, BitSet dimensions) {
+ for (int i = dimensions.nextSetBit(0), index = 0; i >= 0; i = dimensions.nextSetBit(i + 1), index++) {
+ newRecord.set(index, record.get(i));
+ }
+ }
+
private Pair<BitSet, BitSet> getDimensionAndMetricColumBitSet(long cuboidId) {
BitSet bitSet = BitSet.valueOf(new long[]{cuboidId});
BitSet dimension = new BitSet();
@@ -246,20 +277,12 @@ public class CubeStreamBuilder extends StreamBuilder {
);
}
- private void aggregate(Object[] existing, Object[] newRow) {
- aggs.reset();
- aggs.aggregate(existing);
- aggs.aggregate(newRow);
-
- aggs.collectStates(existing);
- }
-
- private String[] buildKey(List<String> row) {
+ private Object[] buildKey(List<String> row, DataTypeSerializer[] serializers) {
int keySize = intermediateTableDesc.getRowKeyColumnIndexes().length;
- String[] key = new String[keySize];
+ Object[] key = new Object[keySize];
for (int i = 0; i < keySize; i++) {
- key[i] = row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]);
+ key[i] = serializers[i].valueOf(row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]));
}
return key;
@@ -274,18 +297,26 @@ public class CubeStreamBuilder extends StreamBuilder {
measureDesc = desc.getMeasures().get(i);
Object value = null;
int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[i];
- if (flatTableIdx == null) {
+ FunctionDesc function = desc.getMeasures().get(i).getFunction();
+ if (function.isCount() || function.isHolisticCountDistinct()) {
+ // note for holistic count distinct, this value will be ignored
+ value = ONE;
+ } else if (flatTableIdx == null) {
value = measureCodec.getSerializer(i).valueOf(measureDesc.getFunction().getParameter().getValue());
} else if (flatTableIdx.length == 1) {
value = measureCodec.getSerializer(i).valueOf(Bytes.toBytes(row.get(flatTableIdx[0])));
} else {
for (int x : flatTableIdx) {
- value = value + row.get(x);
+ if (value == null)
+ value = row.get(x);
+ else
+ value = value + "," + row.get(x);
}
+
+ value = measureCodec.getSerializer(i).valueOf(Bytes.toBytes((String) value));
}
values[i] = value;
}
-
return values;
}
@@ -353,8 +384,4 @@ public class CubeStreamBuilder extends StreamBuilder {
}
- private void submitOffset() {
-
- }
-
}
[24/50] [abbrv] incubator-kylin git commit: KYLIN-650 reiterate port
issue in ODBC tutorial
Posted by li...@apache.org.
KYLIN-650 reiterate port issue in ODBC tutorial
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f22582da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f22582da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f22582da
Branch: refs/heads/streaming-localdict
Commit: f22582da3b7f77b0bab405b2b5d3396322626813
Parents: ae75f5f
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 19:16:12 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 19:16:12 2015 -0700
----------------------------------------------------------------------
docs/Tutorial/Kylin and Tableau Tutorial.md | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f22582da/docs/Tutorial/Kylin and Tableau Tutorial.md
----------------------------------------------------------------------
diff --git a/docs/Tutorial/Kylin and Tableau Tutorial.md b/docs/Tutorial/Kylin and Tableau Tutorial.md
index 8630dab..a2b28b2 100644
--- a/docs/Tutorial/Kylin and Tableau Tutorial.md
+++ b/docs/Tutorial/Kylin and Tableau Tutorial.md
@@ -10,11 +10,13 @@ Kylin and Tableau Tutorial
> More detail information or any issue, please contact Kylin Team: `kylinolap@gmail.com`
### Step 1. Install ODBC Driver
-Refer to wiki page [Kylin ODBC Driver Tutorial](https://github.com/KylinOLAP/Kylin/wiki/Kylin-ODBC-Driver-Tutorial).
+Refer to wiki page [Kylin ODBC Driver Tutorial](Kylin ODBC Driver Tutorial.md).
### Step 2. Connect to Kylin Server
> We recommended to use Connect Using Driver instead of Using DSN since Tableau team will not manage your DSN on Tableau Servers.
+**The snapshots are a little bit outdated because at that time Kylin ues port 9080 as default port, for new comers please use the new default port 7070**
+
Connect Using Driver: Select "Other Database(ODBC)" in the left panel and choose KylinODBCDriver in the pop-up window.
[02/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5b3413b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5b3413b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5b3413b6
Branch: refs/heads/streaming-localdict
Commit: 5b3413b6f0b551f9ff19dea8cf1fb8425e63aac6
Parents: 8f03a27
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 14:30:08 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 14:30:08 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/invertedindex/index/Slice.java | 2 +-
.../invertedindex/model/IIKeyValueCodec.java | 128 +------------------
.../hbase/coprocessor/endpoint/IIEndpoint.java | 12 +-
.../invertedindex/IIStreamBuilder.java | 11 +-
4 files changed, 22 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b3413b6/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
index eb2fb92..7db0a93 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
@@ -42,7 +42,7 @@ public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
final short shard;
final long timestamp;
final int nRecords;
- ColumnValueContainer[] containers;
+ final ColumnValueContainer[] containers;
private Map<Integer, Dictionary<?>> localDictionaries;
public Slice(TableRecordInfoDigest digest, short shard, long timestamp, ColumnValueContainer[] containers) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b3413b6/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index e3284fc..d15f56b 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -65,7 +65,11 @@ public class IIKeyValueCodec implements KeyValueCodec {
ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col);
ImmutableBytesWritable value = container.toBytes();
final Dictionary<?> dictionary = slice.getLocalDictionaries().get(col);
- return new IIRow(key, value, serialize(dictionary));
+ if (dictionary == null) {
+ return new IIRow(key, value, new ImmutableBytesWritable(BytesUtil.EMPTY_BYTE_ARRAY));
+ } else {
+ return new IIRow(key, value, serialize(dictionary));
+ }
}
private static Dictionary<?> deserialize(ImmutableBytesWritable dictBytes) {
@@ -162,17 +166,17 @@ public class IIKeyValueCodec implements KeyValueCodec {
}
int curCol = BytesUtil.readUnsigned(key.get(), i, COLNO_LEN);
- final Dictionary<?> dictionary = deserialize(row.getDictionary());
if (incompleteDigest.isMetrics(curCol)) {
CompressedValueContainer c = new CompressedValueContainer(incompleteDigest, curCol, 0);
c.fromBytes(row.getValue());
valueContainers[curCol] = c;
} else {
+ final Dictionary<?> dictionary = deserialize(row.getDictionary());
CompressedValueContainer c = new CompressedValueContainer(dictionary.getSizeOfId(), dictionary.getMaxId() - dictionary.getMinId() + 1, 0);
c.fromBytes(row.getValue());
valueContainers[curCol] = c;
+ localDictionaries.put(curCol, dictionary);
}
- localDictionaries.put(curCol, dictionary);
columns++;
lastShard = curShard;
lastTimestamp = curTimestamp;
@@ -196,122 +200,4 @@ public class IIKeyValueCodec implements KeyValueCodec {
}
-
- private static class Decoder implements Iterable<Slice> {
-
- private final TableRecordInfoDigest digest;
- Iterator<IIRow> iterator;
-
- Slice slice = null;
- short curShard = Short.MIN_VALUE;
- long curSliceTimestamp = Long.MIN_VALUE;
- int curCol = -1;
- short lastShard = Short.MIN_VALUE;
- long lastSliceTimestamp = Long.MIN_VALUE;
- int lastCol = -1;
- ColumnValueContainer[] containers = null;
- Map<Integer, Dictionary<?>> localDictionaries = Maps.newHashMap();
-
- Decoder(Iterable<IIRow> kvs, TableRecordInfoDigest digest) {
- this.digest = digest;
- this.iterator = kvs.iterator();
- }
-
- private void goToNext() {
- if (slice != null) { // was not fetched
- return;
- }
-
- // NOTE the input keys are ordered
- while (slice == null && iterator.hasNext()) {
- IIRow kv = iterator.next();
- ImmutableBytesWritable k = kv.getKey();
- ImmutableBytesWritable v = kv.getValue();
- decodeKey(k);
- final Dictionary<?> dictionary = deserialize(kv.getDictionary());
- final CompressedValueContainer c = new CompressedValueContainer(digest, curCol, 0);
- c.fromBytes(kv.getValue());
- addContainer(curCol, c);
- localDictionaries.put(curCol, dictionary);
- if (localDictionaries.size() < digest.getColumnCount()) {
- continue;
- }
-
- if (curShard != lastShard
- || curSliceTimestamp != lastSliceTimestamp) {
- makeNext();
- }
- consumeCurrent(v);
- }
- if (slice == null) {
- makeNext();
- }
- }
-
- private void decodeKey(ImmutableBytesWritable k) {
- byte[] buf = k.get();
- int i = k.getOffset();
-
- curShard = (short) BytesUtil.readUnsigned(buf, i, SHARD_LEN);
- i += SHARD_LEN;
- curSliceTimestamp = BytesUtil.readLong(buf, i, TIMEPART_LEN);
- i += TIMEPART_LEN;
-
- curCol = BytesUtil.readUnsigned(buf, i, COLNO_LEN);
- i += COLNO_LEN;
-
- }
-
- private void consumeCurrent(ImmutableBytesWritable v) {
- lastShard = curShard;
- lastSliceTimestamp = curSliceTimestamp;
- lastCol = curCol;
- }
-
- private void makeNext() {
- if (containers != null) {
- slice = new Slice(digest, lastShard, lastSliceTimestamp,
- containers);
- slice.setLocalDictionaries(Maps.newHashMap(localDictionaries));
- }
- lastSliceTimestamp = Long.MIN_VALUE;
- lastCol = -1;
- containers = null;
- localDictionaries.clear();
- }
-
- private void addContainer(int col, ColumnValueContainer c) {
- if (containers == null) {
- containers = new ColumnValueContainer[digest.getColumnCount()];
- }
- containers[col] = c;
- }
-
- @Override
- public Iterator<Slice> iterator() {
- return new Iterator<Slice>() {
- @Override
- public boolean hasNext() {
- goToNext();
- return slice != null;
- }
-
- @Override
- public Slice next() {
- Slice result = slice;
- slice = null;
- return result;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
- }
-
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b3413b6/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index 0e443c9..cb85718 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
@@ -168,8 +169,15 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
byte[] result = new byte[digest.getByteFormLen()];
for (int i = 0; i < coprocessorRowType.getColumnCount(); i++) {
final TblColRef column = coprocessorRowType.columns[i];
- final int length = localDictionaries.get(i).getValueBytesFromId(encodedRecord.getValueID(i), buffer, 0);
- rowKeyColumnIO.writeColumn(column, buffer, length, Dictionary.NULL, result, digest.offset(i));
+ if (digest.isMetrics(i)) {
+ final ImmutableBytesWritable bytes = new ImmutableBytesWritable();
+ encodedRecord.getValueBytes(i, bytes);
+ System.arraycopy(bytes.get(), bytes.getOffset(), buffer, 0, bytes.getLength());
+ rowKeyColumnIO.writeColumn(column, buffer, bytes.getLength(), Dictionary.NULL, result, digest.offset(i));
+ } else {
+ final int length = localDictionaries.get(i).getValueBytesFromId(encodedRecord.getValueID(i), buffer, 0);
+ rowKeyColumnIO.writeColumn(column, buffer, length, Dictionary.NULL, result, digest.offset(i));
+ }
}
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b3413b6/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 480cb39..2ba90fd 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -111,16 +111,13 @@ public class IIStreamBuilder extends StreamBuilder {
private Map<Integer, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
HashMultimap<TblColRef, String> valueMap = HashMultimap.create();
- Set<TblColRef> dimensionColumns = Sets.newHashSet();
- for (int i = 0; i < desc.listAllColumns().size(); i++) {
- if (!desc.isMetricsCol(i)) {
- dimensionColumns.add(desc.listAllColumns().get(i));
- }
- }
+ final List<TblColRef> allColumns = desc.listAllColumns();
for (List<String> row : table) {
for (int i = 0; i < row.size(); i++) {
String cell = row.get(i);
- valueMap.put(desc.listAllColumns().get(i), cell);
+ if (!desc.isMetricsCol(i)) {
+ valueMap.put(allColumns.get(i), cell);
+ }
}
}
Map<Integer, Dictionary<?>> result = Maps.newHashMap();
[45/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a7b771d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a7b771d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a7b771d1
Branch: refs/heads/streaming-localdict
Commit: a7b771d1ca33995e8db0f9629036ce6659c66b5c
Parents: dda0e33 03eb771
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 20 16:55:57 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 20 16:55:57 2015 +0800
----------------------------------------------------------------------
streaming/.settings/org.eclipse.core.resources.prefs | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
[40/50] [abbrv] incubator-kylin git commit: KYLIN-648, refine test sql
Posted by li...@apache.org.
KYLIN-648, refine test sql
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/3563a914
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/3563a914
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/3563a914
Branch: refs/heads/streaming-localdict
Commit: 3563a91469619dc1ca22816175e26e22557694ce
Parents: c66a74b
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 20 15:31:37 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 20 15:31:37 2015 +0800
----------------------------------------------------------------------
query/src/test/resources/query/sql/query78.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3563a914/query/src/test/resources/query/sql/query78.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query78.sql b/query/src/test/resources/query/sql/query78.sql
index f1d8fb2..0da9611 100644
--- a/query/src/test/resources/query/sql/query78.sql
+++ b/query/src/test/resources/query/sql/query78.sql
@@ -1,4 +1,4 @@
-select sum(PRICE), LSTG_FORMAT_NAME
+select sum(PRICE) as GMV, LSTG_FORMAT_NAME
from test_kylin_fact
where (LSTG_FORMAT_NAME in ('ABIN')) or (LSTG_FORMAT_NAME>='FP-GTC' and LSTG_FORMAT_NAME<='Others')
group by LSTG_FORMAT_NAME
[48/50] [abbrv] incubator-kylin git commit: fix bug
Posted by li...@apache.org.
fix bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/21909b9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/21909b9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/21909b9a
Branch: refs/heads/streaming-localdict
Commit: 21909b9aacccc35e1bd013242d60dccfc3efcfdd
Parents: a7b771d
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 20 18:47:48 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 20 18:47:48 2015 +0800
----------------------------------------------------------------------
.../kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/21909b9a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index cb85718..c166e73 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -189,9 +189,10 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
ClearTextDictionary clearTextDictionary = new ClearTextDictionary(recordInfo, type);
RowKeyColumnIO rowKeyColumnIO = new RowKeyColumnIO(clearTextDictionary);
for (Slice slice : slices) {
+ CoprocessorFilter newFilter = CoprocessorFilter.fromFilter(new LocalDictionary(slice.getLocalDictionaries(), type, slice.getInfo()), filter.getFilter(), FilterDecorator.FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT);
ConciseSet result = null;
if (filter != null) {
- result = new BitMapFilterEvaluator(new SliceBitMapProvider(slice, type)).evaluate(filter.getFilter());
+ result = new BitMapFilterEvaluator(new SliceBitMapProvider(slice, type)).evaluate(newFilter.getFilter());
}
Iterator<RawTableRecord> iterator = slice.iterateWithBitmap(result);
[44/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dda0e333
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dda0e333
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dda0e333
Branch: refs/heads/streaming-localdict
Commit: dda0e3337abd39eb70645e8882a616cac116bfc8
Parents: 4a5c9ce
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 20 16:55:48 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 20 16:55:48 2015 +0800
----------------------------------------------------------------------
.../invertedindex/index/BatchSliceBuilder.java | 100 +++++++++++++++++++
.../invertedindex/model/IIKeyValueCodec.java | 2 +-
.../apache/kylin/job/BuildIIWithStreamTest.java | 22 +++-
.../invertedindex/IIStreamBuilder.java | 37 ++++---
4 files changed, 140 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dda0e333/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceBuilder.java
new file mode 100644
index 0000000..94b70c1
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceBuilder.java
@@ -0,0 +1,100 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.invertedindex.index;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.kylin.invertedindex.model.IIDesc;
+
+import java.util.List;
+
+/**
+ * Created by qianzhou on 3/20/15.
+ */
+public class BatchSliceBuilder {
+
+ private final int nColumns;
+ private final int nRecordsCap;
+ private final short shard;
+ private final IIDesc desc;
+
+ private long sliceTimestamp;
+
+ transient ImmutableBytesWritable temp = new ImmutableBytesWritable();
+
+ public BatchSliceBuilder(IIDesc desc, short shard) {
+ this.desc = desc;
+ this.nColumns = desc.listAllColumns().size();
+ this.nRecordsCap = Math.max(1, desc.getSliceSize());
+
+ this.shard = shard;
+ this.sliceTimestamp = Long.MIN_VALUE;
+ }
+
+ public Slice build(TableRecordInfoDigest digest, List<TableRecord> records) {
+ Preconditions.checkArgument(records != null && !records.isEmpty(), "records cannot be empty");
+ Preconditions.checkArgument(records.size() <= nRecordsCap, "batch count cannot exceed " + nRecordsCap);
+ sliceTimestamp = increaseSliceTimestamp(records.get(0).getTimestamp());
+ ColumnValueContainer[] containers = new ColumnValueContainer[nColumns];
+ for (int i : desc.getValueColumns()) {
+ containers[i] = new CompressedValueContainer(digest, i, nRecordsCap);
+ }
+ for (int i : desc.getMetricsColumns()) {
+ containers[i] = new CompressedValueContainer(digest, i, nRecordsCap);
+ }
+ for (TableRecord record : records) {
+ Preconditions.checkArgument(record.getShard() == shard, "record shard:" + record.getShard() + " but should be " + shard);
+ for (int i = 0; i < nColumns; i++) {
+ record.getValueBytes(i, temp);
+ containers[i].append(temp);
+ }
+ }
+ return new Slice(digest, shard, sliceTimestamp, containers);
+
+ }
+
+ private long increaseSliceTimestamp(long timestamp) {
+ if (timestamp < sliceTimestamp) {
+ throw new IllegalStateException();
+ }
+
+ if (timestamp == sliceTimestamp) {
+ return ++timestamp; // ensure slice timestamp increases
+ } else {
+ return timestamp;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dda0e333/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index ee26181..eedda4b 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -181,7 +181,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
short lastShard = 0;
long lastTimestamp = 0;
- while (iterator().hasNext() && columns < incompleteDigest.getColumnCount()) {
+ while (iterator.hasNext() && columns < incompleteDigest.getColumnCount()) {
final IIRow row = iterator.next();
final ImmutableBytesWritable key = row.getKey();
int i = key.getOffset();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dda0e333/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 8c8d580..652fca0 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -34,6 +34,7 @@
package org.apache.kylin.job;
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ToolRunner;
@@ -68,9 +69,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -216,9 +215,24 @@ public class BuildIIWithStreamTest {
ExecutorService executorService = Executors.newSingleThreadExecutor();
final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, segment.getStorageLocationIdentifier(), desc, 0);
+ int count = 0;
+ List<String[]> rawData = Lists.newArrayList();
while (reader.next()) {
- queue.put(parse(reader.getRow()));
+ desc.getTimestampColumn();
+ rawData.add(reader.getRow());
+ count++;
}
+ final int timestampColumn = desc.getTimestampColumn();
+ Collections.sort(rawData, new Comparator<String[]>() {
+ @Override
+ public int compare(String[] o1, String[] o2) {
+ return o1[timestampColumn].compareTo(o2[timestampColumn]);
+ }
+ });
+ for (String[] row : rawData) {
+ queue.put(parse(row));
+ }
+ logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
queue.put(new Stream(-1, null));
final Future<?> future = executorService.submit(streamBuilder);
try {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dda0e333/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 2ba90fd..2e5f0d1 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -36,7 +36,10 @@ package org.apache.kylin.streaming.invertedindex;
import com.google.common.base.Function;
import com.google.common.base.Stopwatch;
-import com.google.common.collect.*;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
@@ -44,8 +47,8 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.invertedindex.index.BatchSliceBuilder;
import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.SliceBuilder;
import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIDesc;
@@ -61,7 +64,6 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
@@ -72,20 +74,20 @@ public class IIStreamBuilder extends StreamBuilder {
private static Logger logger = LoggerFactory.getLogger(IIStreamBuilder.class);
- private IIDesc desc = null;
- private HTableInterface hTable = null;
- private int partitionId = -1;
+ private final IIDesc desc;
+ private final HTableInterface hTable;
+ private final BatchSliceBuilder sliceBuilder;
public IIStreamBuilder(LinkedBlockingDeque<Stream> queue, String hTableName, IIDesc desc, int partitionId) {
super(queue, desc.getSliceSize());
this.desc = desc;
- this.partitionId = partitionId;
try {
this.hTable = HConnectionManager.createConnection(HBaseConfiguration.create()).getTable(hTableName);
} catch (IOException e) {
logger.error("cannot open htable name:" + hTableName, e);
throw new RuntimeException("cannot open htable name:" + hTableName, e);
}
+ sliceBuilder = new BatchSliceBuilder(desc, (short) partitionId);
}
@Override
@@ -101,8 +103,8 @@ public class IIStreamBuilder extends StreamBuilder {
});
final Map<Integer, Dictionary<?>> dictionaryMap = buildDictionary(table, desc);
TableRecordInfo tableRecordInfo = new TableRecordInfo(desc, dictionaryMap);
- SliceBuilder sliceBuilder = new SliceBuilder(tableRecordInfo, (short) partitionId);
final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo, dictionaryMap);
+ logger.info("slice info, shard:" + slice.getShard() + " timestamp:" + slice.getTimestamp() + " record count:" + slice.getRecordCount());
loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()));
submitOffset();
stopwatch.stop();
@@ -137,15 +139,18 @@ public class IIStreamBuilder extends StreamBuilder {
return Lists.newArrayList(new String(stream.getRawData()).split(","));
}
- private Slice buildSlice(List<List<String>> table, SliceBuilder sliceBuilder, TableRecordInfo tableRecordInfo, Map<Integer, Dictionary<?>> localDictionary) {
- for (List<String> row : table) {
- TableRecord tableRecord = tableRecordInfo.createTableRecord();
- for (int i = 0; i < row.size(); i++) {
- tableRecord.setValueString(i, row.get(i));
+ private Slice buildSlice(List<List<String>> table, BatchSliceBuilder sliceBuilder, final TableRecordInfo tableRecordInfo, Map<Integer, Dictionary<?>> localDictionary) {
+ final Slice slice = sliceBuilder.build(tableRecordInfo.getDigest(), Lists.transform(table, new Function<List<String>, TableRecord>() {
+ @Nullable
+ @Override
+ public TableRecord apply(@Nullable List<String> input) {
+ TableRecord result = tableRecordInfo.createTableRecord();
+ for (int i = 0; i < input.size(); i++) {
+ result.setValueString(i, input.get(i));
+ }
+ return result;
}
- sliceBuilder.append(tableRecord);
- }
- final Slice slice = sliceBuilder.close();
+ }));
slice.setLocalDictionaries(localDictionary);
return slice;
}
[43/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/03eb7712
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/03eb7712
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/03eb7712
Branch: refs/heads/streaming-localdict
Commit: 03eb7712010c8e1be30c6fe4d5fbb63ab7875742
Parents: 46355ed 4a5c9ce
Author: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Authored: Fri Mar 20 09:05:29 2015 +0100
Committer: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Committed: Fri Mar 20 09:05:29 2015 +0100
----------------------------------------------------------------------
.../kylin/metadata/filter/CompareTupleFilter.java | 4 +++-
.../coprocessor/endpoint/EndpointTupleIterator.java | 2 +-
.../apache/kylin/storage/filter/FilterBaseTest.java | 8 ++++++++
.../kylin/storage/filter/FilterSerializeTest.java | 16 ++++++++++++++++
4 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[39/50] [abbrv] incubator-kylin git commit: update eclipse settings
Posted by li...@apache.org.
update eclipse settings
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/46355edd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/46355edd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/46355edd
Branch: refs/heads/streaming-localdict
Commit: 46355edd8107cbc6036bb4302dc29898d030046d
Parents: 696a673
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 20 15:26:52 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 20 15:26:52 2015 +0800
----------------------------------------------------------------------
streaming/.settings/org.eclipse.core.resources.prefs | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/46355edd/streaming/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/streaming/.settings/org.eclipse.core.resources.prefs b/streaming/.settings/org.eclipse.core.resources.prefs
index 839d647..29abf99 100644
--- a/streaming/.settings/org.eclipse.core.resources.prefs
+++ b/streaming/.settings/org.eclipse.core.resources.prefs
@@ -2,4 +2,5 @@ eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
+encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8
[15/50] [abbrv] incubator-kylin git commit: fix export result bug
Posted by li...@apache.org.
fix export result bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8a0a0ac5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8a0a0ac5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8a0a0ac5
Branch: refs/heads/streaming-localdict
Commit: 8a0a0ac58cc9e35d6d50a7d3eb66370e68c565ba
Parents: 624c434
Author: jiazhong <ji...@ebay.com>
Authored: Thu Mar 19 17:45:05 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Thu Mar 19 17:45:05 2015 +0800
----------------------------------------------------------------------
webapp/app/partials/query/query_detail.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8a0a0ac5/webapp/app/partials/query/query_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/query/query_detail.html b/webapp/app/partials/query/query_detail.html
index a4c11bf..7157e9c 100644
--- a/webapp/app/partials/query/query_detail.html
+++ b/webapp/app/partials/query/query_detail.html
@@ -88,7 +88,7 @@
<span ng-if="curQuery.graph.show"><i class="fa fa-list-ul"></i> Grid</span>
</button>
<a class="btn btn-default btn-xs"
- href="{{config.service.url}}query/format/csv?sql={{sanitate(curQuery.sql)}}&project={{state.selectedProject}}" target="_blank"><i
+ href="{{config.service.url}}query/format/csv?sql={{sanitate(curQuery.sql)}}&project={{curQuery.project}}" target="_blank"><i
class="fa fa-cloud-download"></i> Export</a>
<a class="btn btn-default btn-xs"
ng-click="refreshUi();"><i
[37/50] [abbrv] incubator-kylin git commit: fix bug
Posted by li...@apache.org.
fix bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/55c8257e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/55c8257e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/55c8257e
Branch: refs/heads/streaming-localdict
Commit: 55c8257ef5b9ca8aeae10e2edfe49fcf4226349b
Parents: 696a673
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 20 15:21:23 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 20 15:21:23 2015 +0800
----------------------------------------------------------------------
.../coprocessor/endpoint/EndpointTupleIterator.java | 2 +-
.../apache/kylin/storage/filter/FilterBaseTest.java | 8 ++++++++
.../kylin/storage/filter/FilterSerializeTest.java | 16 ++++++++++++++++
3 files changed, 25 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/55c8257e/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index cb543c7..86736ba 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -119,7 +119,7 @@ public class EndpointTupleIterator implements ITupleIterator {
this.tableRecordInfo = new TableRecordInfo(this.seg);
this.pushedDownRowType = CoprocessorRowType.fromTableRecordInfo(tableRecordInfo, this.columns);
- this.pushedDownFilter = CoprocessorFilter.fromFilter(this.seg, rootFilter, FilterDecorator.FilterConstantsTreatment.AS_IT_IS);
+ this.pushedDownFilter = CoprocessorFilter.fromFilter(new ClearTextDictionary(this.tableRecordInfo), rootFilter, FilterDecorator.FilterConstantsTreatment.AS_IT_IS);
for (TblColRef column : this.pushedDownFilter.getUnstrictlyFilteredColumns()) {
groupBy.add(column);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/55c8257e/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
index 95c6555..87e216a 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
@@ -136,6 +136,14 @@ public class FilterBaseTest {
return compareFilter;
}
+ protected CompareTupleFilter buildCompareDynamicFilter(List<TblColRef> groups) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
+ compareFilter.addChild(new ColumnTupleFilter(groups.get(0)));
+ compareFilter.addChild(new DynamicTupleFilter("?0"));
+ compareFilter.bindVariable("?0", "abc");
+ return compareFilter;
+ }
+
protected void compareFilter(TupleFilter f1, TupleFilter f2) {
if (f1 == null && f2 == null) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/55c8257e/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
index a095012..f2c531a 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.storage.filter;
import java.util.ArrayList;
import java.util.List;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
@@ -202,4 +203,19 @@ public class FilterSerializeTest extends FilterBaseTest {
compareFilter(filter, newFilter);
}
+ @Test
+ public void testDynamic() {
+ final CompareTupleFilter compareDynamicFilter = buildCompareDynamicFilter(buildGroups());
+
+ byte[] bytes = TupleFilterSerializer.serialize(compareDynamicFilter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ bytes = TupleFilterSerializer.serialize(newFilter, CS);
+ TupleFilter newFilter2 = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(compareDynamicFilter, newFilter);
+ compareFilter(compareDynamicFilter, newFilter2);
+
+ }
+
}
[27/50] [abbrv] incubator-kylin git commit: KYLIN-650 add maillist
Posted by li...@apache.org.
KYLIN-650 add maillist
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/eff13931
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/eff13931
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/eff13931
Branch: refs/heads/streaming-localdict
Commit: eff1393143a8795b7b08c1ba05d564c914cb09f1
Parents: 7faa216
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 19:30:17 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 19:30:17 2015 -0700
----------------------------------------------------------------------
README.md | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eff13931/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 28c2df6..232758c 100644
--- a/README.md
+++ b/README.md
@@ -17,15 +17,18 @@ Tutorial
Please follow this installation tutorial to start with Kylin: [Installation Tutorial](docs/Installation/Installation.md)
+Get Help
+------------
+
+The fastest way to get response from our developers is to send email to our mail list <de...@kylin.incubator.apache.org>, and remember to subscribe our mail list via <de...@kylin.incubator.apache.org>
+
Resources
------------
* Web Site: <http://kylin.io>
-* Google Group: [Kylin OLAP Group](https://groups.google.com/forum/#!forum/kylin-olap)
-
-* Developer Mail: <ky...@googlegroups.com>
+* Developer Mail: <de...@kylin.incubator.apache.org>
* How To Contribute: See [wiki](https://github.com/KylinOLAP/Kylin/wiki/How-to-Contribute)
[10/50] [abbrv] incubator-kylin git commit: Merge branch
'streaming-localdict' of https://github.com/KylinOLAP/Kylin into
streaming-localdict
Posted by li...@apache.org.
Merge branch 'streaming-localdict' of https://github.com/KylinOLAP/Kylin into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/12365005
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/12365005
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/12365005
Branch: refs/heads/streaming-localdict
Commit: 12365005ccbf034df7df7ef3eb5159dfbf62c5aa
Parents: 6fded26 db3cb94
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 18:54:37 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 18:54:37 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ByteArray.java | 6 +
.../apache/kylin/storage/gridtable/GTInfo.java | 57 +++++-
.../storage/gridtable/GTInvertedIndex.java | 201 +++++++++++++++++++
.../gridtable/GTInvertedIndexOfColumn.java | 116 +++++++++++
.../kylin/storage/gridtable/GTRawScanner.java | 16 +-
.../kylin/storage/gridtable/GTRecord.java | 12 +-
.../storage/gridtable/GTRowBlockIndex.java | 5 -
.../storage/gridtable/GTSampleCodeSystem.java | 8 +-
.../apache/kylin/storage/gridtable/GTUtil.java | 23 +++
.../storage/gridtable/GTInvertedIndexTest.java | 165 +++++++++++++++
.../kylin/storage/gridtable/GridTableTest.java | 9 +-
11 files changed, 581 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
[33/50] [abbrv] incubator-kylin git commit: KYLIN-650 Kylin Document
Convensions
Posted by li...@apache.org.
KYLIN-650 Kylin Document Convensions
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a8757d57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a8757d57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a8757d57
Branch: refs/heads/streaming-localdict
Commit: a8757d576083d69ee996ae75c63bfae813aedbb4
Parents: af42057
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 23:36:55 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 23:36:55 2015 -0700
----------------------------------------------------------------------
README.md | 7 +++++--
docs/MISC/Kylin Document Convensions.md | 8 ++++++++
2 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a8757d57/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index fba7c0a..39bb3e1 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ Tutorial
Please follow this installation tutorial to start with Kylin: [Installation Tutorial](docs/Installation/Installation.md)
-Advance Topics
+Advanced Topics
-------
####Connectivity
1.[How to use kylin remote jdbc driver](docs/Tutorial/How to use kylin remote jdbc driver.md)
@@ -67,5 +67,8 @@ Resources
* How To Contribute: See [this](docs/MISC/How to Contribute.md)
-* Apache Proposal: [Apache Kylin](https://wiki.apache.org/incubator/KylinProposal)
+* Apache Proposal: [Apache Kylin](https://wiki.apache.org/incubator/KylinProposal)
+
+* [Kylin Document Convensions](docs/MISC/Kylin Document Convensions.md)
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a8757d57/docs/MISC/Kylin Document Convensions.md
----------------------------------------------------------------------
diff --git a/docs/MISC/Kylin Document Convensions.md b/docs/MISC/Kylin Document Convensions.md
new file mode 100644
index 0000000..0d45dd9
--- /dev/null
+++ b/docs/MISC/Kylin Document Convensions.md
@@ -0,0 +1,8 @@
+Kylin Document Convernsions
+-----
+
+1. Keep documentation in markdown files
+2. In markdown files, try to use relative URI for document/image reference.
+3. Never link to github wiki pages
+4. Try not to simply list all docuemnts in one page. Instead, try to organize different documents into a "story", so that readers are presented deeper materials as they proceed.
+5. ...
\ No newline at end of file
[12/50] [abbrv] incubator-kylin git commit: refactor
Posted by li...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/59253205
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/59253205
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/59253205
Branch: refs/heads/streaming-localdict
Commit: 59253205d9fc4548e843a84e4ee5e0e7f7b39b1a
Parents: 1a46c2d
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 20:18:56 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 20:18:56 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/streaming/KafkaConsumer.java | 15 +++++----------
.../apache/kylin/streaming/KafkaConsumerTest.java | 12 +++++++++++-
2 files changed, 16 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59253205/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
index fb0aba6..42a0f1f 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
@@ -51,7 +51,7 @@ import java.util.concurrent.atomic.AtomicLong;
/**
* Created by qianzhou on 2/15/15.
*/
-public class KafkaConsumer implements Runnable {
+public abstract class KafkaConsumer implements Runnable {
private String topic;
private int partitionId;
@@ -111,16 +111,9 @@ public class KafkaConsumer implements Runnable {
continue;
}
for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(topic, partitionId)) {
- final ByteBuffer payload = messageAndOffset.message().payload();
-
- //TODO use ByteBuffer maybe
- byte[] bytes = new byte[payload.limit()];
- payload.get(bytes);
- logger.info("get message offset:" + messageAndOffset.offset());
- logger.info(new String(bytes));
try {
- streamQueue.put(new Stream(messageAndOffset.offset(), bytes));
- } catch (InterruptedException e) {
+ consume(messageAndOffset.offset(), messageAndOffset.message().payload());
+ } catch (Exception e) {
logger.error("error put streamQueue", e);
break;
}
@@ -132,4 +125,6 @@ public class KafkaConsumer implements Runnable {
}
}
+ protected abstract void consume(long offset, ByteBuffer payload) throws Exception;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59253205/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
index acae5bd..be65a4f 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
@@ -40,6 +40,7 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
@@ -83,7 +84,16 @@ public class KafkaConsumerTest extends KafkaBaseTest {
final ExecutorService executorService = Executors.newFixedThreadPool(kafkaTopicMeta.getPartitionIds().size());
List<BlockingQueue<Stream>> queues = Lists.newArrayList();
for (Integer partitionId : kafkaTopicMeta.getPartitionIds()) {
- KafkaConsumer consumer = new KafkaConsumer(kafkaTopicMeta.getName(), partitionId, kafkaConfig.getBrokers(), kafkaConfig);
+ KafkaConsumer consumer = new KafkaConsumer(kafkaTopicMeta.getName(), partitionId, kafkaConfig.getBrokers(), kafkaConfig) {
+ @Override
+ protected void consume(long offset, ByteBuffer payload) throws Exception {
+ //TODO use ByteBuffer maybe
+ byte[] bytes = new byte[payload.limit()];
+ payload.get(bytes);
+ logger.info("get message offset:" + offset);
+ getStreamQueue().put(new Stream(offset, bytes));
+ }
+ };
queues.add(consumer.getStreamQueue());
executorService.execute(consumer);
}
[23/50] [abbrv] incubator-kylin git commit: Merge remote-tracking
branch 'origin/streaming' into streaming-localdict
Posted by li...@apache.org.
Merge remote-tracking branch 'origin/streaming' into streaming-localdict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/84ec8e0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/84ec8e0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/84ec8e0e
Branch: refs/heads/streaming-localdict
Commit: 84ec8e0ecfd13d956cd53b46f990f16a8950259b
Parents: c952a36 8a0a0ac
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 20 10:16:07 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 20 10:16:07 2015 +0800
----------------------------------------------------------------------
webapp/app/partials/query/query_detail.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[41/50] [abbrv] incubator-kylin git commit: KYLIN-648, refine test sql
Posted by li...@apache.org.
KYLIN-648, refine test sql
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/222b1440
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/222b1440
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/222b1440
Branch: refs/heads/streaming-localdict
Commit: 222b1440a83a1990c56b2441b0fb3a905204510c
Parents: c66a74b
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 20 15:31:37 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 20 15:44:28 2015 +0800
----------------------------------------------------------------------
query/src/test/resources/query/sql/query78.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/222b1440/query/src/test/resources/query/sql/query78.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query78.sql b/query/src/test/resources/query/sql/query78.sql
index f1d8fb2..d055b8e 100644
--- a/query/src/test/resources/query/sql/query78.sql
+++ b/query/src/test/resources/query/sql/query78.sql
@@ -1,4 +1,4 @@
-select sum(PRICE), LSTG_FORMAT_NAME
+select sum(PRICE) as GMV, LSTG_FORMAT_NAME as FORMAT_NAME
from test_kylin_fact
where (LSTG_FORMAT_NAME in ('ABIN')) or (LSTG_FORMAT_NAME>='FP-GTC' and LSTG_FORMAT_NAME<='Others')
group by LSTG_FORMAT_NAME
[31/50] [abbrv] incubator-kylin git commit: KYLIN-650 category
resting pages to different topics
Posted by li...@apache.org.
KYLIN-650 category resting pages to different topics
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5acdcd23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5acdcd23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5acdcd23
Branch: refs/heads/streaming-localdict
Commit: 5acdcd23098dda31338ebdb687bc7fc86076b160
Parents: 284ef26
Author: honma <ho...@ebay.com>
Authored: Thu Mar 19 23:23:11 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 19 23:23:11 2015 -0700
----------------------------------------------------------------------
README.md | 15 +
docs/REST/Build Cube with Restful API.md | 51 +
...w to consume Kylin REST API in javascript.md | 42 +
docs/REST/Kylin Restful API List.md | 2020 ++++++++++++++++++
.../TechInside/New meta data model structure.md | 18 +
...n Kylin test case with HBase Mini Cluster.md | 15 +
.../How to use kylin remote jdbc driver.md | 83 +
7 files changed, 2244 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5acdcd23/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index fa180b0..f3d383f 100644
--- a/README.md
+++ b/README.md
@@ -19,8 +19,19 @@ Please follow this installation tutorial to start with Kylin: [Installation Tuto
Advance Topics
-------
+####Connectivity
+1.[How to use kylin remote jdbc driver](docs/Tutorial/How to use kylin remote jdbc driver.md)
+
+2.[Kylin ODBC Driver Tutorial](docs/Tutorial/Kylin ODBC Driver Tutorial.md)
+
####REST
+1.[Kylin Restful API List](docs/REST/Kylin Restful API List.md)
+
+2.[Build Cube with Restful API](docs/REST/Build Cube with Restful API.md)
+
+3.[How to consume Kylin REST API in javascript](docs/REST/How to consume Kylin REST API in javascript.md)
+
####Operations
1.[Kylin Metadata Store](docs/Operations/Kylin Metadata Store.md)
@@ -30,8 +41,12 @@ Advance Topics
####Test
+1.[Run Kylin test case with HBase Mini Cluster](docs/Test/Run Kylin test case with HBase Mini Cluster.md)
+
+
####Technial Details
+1.[New meta data model structure](docs/TechInside/New meta data model structure.md)
Get Help
------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5acdcd23/docs/REST/Build Cube with Restful API.md
----------------------------------------------------------------------
diff --git a/docs/REST/Build Cube with Restful API.md b/docs/REST/Build Cube with Restful API.md
new file mode 100644
index 0000000..4eb5381
--- /dev/null
+++ b/docs/REST/Build Cube with Restful API.md
@@ -0,0 +1,51 @@
+
+### 1. Authentication
+* Currently, Kylin uses [basic authentication](http://en.wikipedia.org/wiki/Basic_access_authentication).
+* Add `Authorization` header to first request for authentication
+* Or you can do a specific request by `POST http://localhost:7070/kylin/api/user/authentication`
+* Once authenticated, client can go subsequent requests with cookies.
+* Example
+```
+ POST http://localhost:7070/kylin/api/user/authentication
+
+ Authorization:Basic xxxxJD124xxxGFxxxSDF
+ Content-Type: application/json;charset=UTF-8
+```
+
+### 2. Get details of cube.
+* `GET http://localhost:7070/kylin/api/cubes?cubeName={cube_name}&limit=15&offset=0`
+* Client can find cube segment date ranges in returned cube detail.
+* Example
+```
+ GET http://localhost:7070/kylin/api/cubes?cubeName=test_kylin_cube_with_slr&limit=15&offset=0
+
+ Authorization:Basic xxxxJD124xxxGFxxxSDF
+ Content-Type: application/json;charset=UTF-8
+```
+
+### 3. Then submit a build job of the cube.
+* `PUT http://localhost:7070/kylin/api/cubes/{cube_name}/rebuild`
+* For put request body detail please refer to [service doc](https://github.com/KylinOLAP/Kylin/wiki/Restful-Service-Doc).
+ * `startTime` and `endTime` should be utc timestamp.
+ * `buildType` can be `BUILD` or `MERGE`. `BUILD` is for building a new segment or refreshing an existing segment. `MERGE` is for merging multiple existing segments into one bigger segment.
+* This method will return a newly created job instance, in which the uuid is the identity of job to track job status.
+* Example
+```
+ PUT http://localhost:7070/kylin/api/cubes/test_kylin_cube_with_slr/rebuild
+
+ Authorization:Basic xxxxJD124xxxGFxxxSDF
+ Content-Type: application/json;charset=UTF-8
+
+ {
+ "startTime": 0,
+ "endTime": 1388563200000,
+ "buildType": "BUILD"
+ }
+```
+
+### 4. Track job status.
+* `GET http://localhost:7070/kylin/api/jobs/{job_uuid}`
+* Returned `job_status` represents current status of job.
+
+### 5. If the job failed, you can resume the job.
+* `PUT http://localhost:7070/kylin/api/jobs/{job_uuid}/resume`
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5acdcd23/docs/REST/How to consume Kylin REST API in javascript.md
----------------------------------------------------------------------
diff --git a/docs/REST/How to consume Kylin REST API in javascript.md b/docs/REST/How to consume Kylin REST API in javascript.md
new file mode 100644
index 0000000..a0ea1b6
--- /dev/null
+++ b/docs/REST/How to consume Kylin REST API in javascript.md
@@ -0,0 +1,42 @@
+Kylin securicty authorization is based on basic access authorization, so when you want to use API in your javascript,you need to involve the authorization info in http headers.
+
+
+## Here is a demo show how to use Kylin Query API.
+```
+$.ajaxSetup({
+ headers: { 'Authorization': "Basic eWFu**********X***ZA==", 'Content-Type': 'application/json;charset=utf-8' } // use your own authorization code here
+ });
+ var request = $.ajax({
+ url: "http://hostname/kylin/api/query",
+ type: "POST",
+ data: '{"sql":"select count(*) from SUMMARY;","offset":0,"limit":50000,"partial":true,"project":"test"}',
+ dataType: "json"
+ });
+ request.done(function( msg ) {
+ alert(msg);
+ });
+ request.fail(function( jqXHR, textStatus ) {
+ alert( "Request failed: " + textStatus );
+ });
+
+```
+
+
+For what is basic access authorization ,click here http://en.wikipedia.org/wiki/Basic_access_authentication
+
+## Keypoint:
+1. add basic access authorization info in heders.
+2. use right ajax type and right data synax
+
+How to generate your authorization code (download and import "jquery.base64.js" from https://github.com/yckart/jquery.base64.js)
+
+```
+var authorizationCode = $.base64('encode', 'NT_USERNAME' + ":" + 'NT_PASSWORD');
+
+$.ajaxSetup({
+ headers: {
+ 'Authorization': "Basic " + authorizationCode,
+ 'Content-Type': 'application/json;charset=utf-8'
+ }
+});
+```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5acdcd23/docs/REST/Kylin Restful API List.md
----------------------------------------------------------------------
diff --git a/docs/REST/Kylin Restful API List.md b/docs/REST/Kylin Restful API List.md
new file mode 100644
index 0000000..f1398e9
--- /dev/null
+++ b/docs/REST/Kylin Restful API List.md
@@ -0,0 +1,2020 @@
+## Kylin Restful API List
+
+* QUERY
+ * Login
+ * Query
+ * List queryable tables
+* CUBE
+ * List cubes
+ * Rebuild cube
+ * Disable cube
+ * Purge cube
+ * Enable cube
+* JOB
+ * Resume job
+ * Discard job
+ * Get job step log
+* Metadata
+ * Get Hive Table
+ * Get Hive Table (Extend Info)
+ * Get Hive Tables
+ * Load Hive Table
+* Wipe cache
+
+***
+
+### Login
+`POST /user/authentication`
+
+for example:
+```sh
+curl -c /path/to/cookiefile.txt -X POST -H "Authorization: Basic XXXXXXXXX" -H 'Content-Type: application/json' http://<host>:<port>/kylin/api/user/authentication
+```
+
+If login successfully, the JSESSIONID will be saved into the cookie file; In the subsequent http requests, attach the cookie, for example:
+```sh
+curl -b /path/to/cookiefile.txt -X PUT -H 'Content-Type: application/json' -d '{"startTime":'1423526400000', "endTime":'1423526400', "buildType":"BUILD"}' http://<host>:<port>/kylin/api/cubes/your_cube/rebuild
+```
+
+#### Header
+Authorization data encoded by basic auth. Header sample:
+Authorization:Basic {data}
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ "userDetails":{
+ "password":null,
+ "username":"sample",
+ "authorities":[
+ {
+ "authority":"ROLE_ANALYST"
+ },
+ {
+ "authority":"ROLE_MODELER"
+ }
+ ],
+ "accountNonExpired":true,
+ "accountNonLocked":true,
+ "credentialsNonExpired":true,
+ "enabled":true
+ }
+}
+```
+
+***
+
+### Query
+`POST /query`
+#### Request Body
+* sql - `required` `string` The sql query string.
+* offset - `optional` `int` Query offset. If offset is set in sql, curIndex will be ignored.
+* limit - `optional` `int` Query limit. If limit is set in sql, perPage will be ignored.
+* acceptPartial - `optional` `bool` Flag to switch partial query.
+* project - `optional` `string` Project to perform query. Default value is 'DEFAULT'
+```sh
+{
+ "sql":"select * from TEST_KYLIN_FACT",
+ "offset":0,
+ "limit":50000,
+ "acceptPartial":true,
+ "project":"DEFAULT"
+}
+```
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ "columnMetas":[
+ {
+ "isNullable":1,
+ "displaySize":0,
+ "label":"CAL_DT",
+ "name":"CAL_DT",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":0,
+ "scale":0,
+ "columnType":91,
+ "columnTypeName":"DATE",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ },
+ {
+ "isNullable":1,
+ "displaySize":10,
+ "label":"LEAF_CATEG_ID",
+ "name":"LEAF_CATEG_ID",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":10,
+ "scale":0,
+ "columnType":4,
+ "columnTypeName":"INTEGER",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ },
+ {
+ "isNullable":1,
+ "displaySize":10,
+ "label":"LSTG_SITE_ID",
+ "name":"LSTG_SITE_ID",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":10,
+ "scale":0,
+ "columnType":4,
+ "columnTypeName":"INTEGER",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ },
+ {
+ "isNullable":1,
+ "displaySize":5,
+ "label":"SLR_SEGMENT_CD",
+ "name":"SLR_SEGMENT_CD",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":5,
+ "scale":0,
+ "columnType":5,
+ "columnTypeName":"SMALLINT",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ },
+ {
+ "isNullable":1,
+ "displaySize":256,
+ "label":"LSTG_FORMAT_NAME",
+ "name":"LSTG_FORMAT_NAME",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":256,
+ "scale":0,
+ "columnType":12,
+ "columnTypeName":"VARCHAR(256)",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ },
+ {
+ "isNullable":1,
+ "displaySize":19,
+ "label":"SELLER_ID",
+ "name":"SELLER_ID",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":19,
+ "scale":0,
+ "columnType":-5,
+ "columnTypeName":"BIGINT",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ },
+ {
+ "isNullable":1,
+ "displaySize":19,
+ "label":"PRICE",
+ "name":"PRICE",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":19,
+ "scale":4,
+ "columnType":3,
+ "columnTypeName":"DECIMAL(19, 4)",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ },
+ {
+ "isNullable":1,
+ "displaySize":19,
+ "label":"MIN_PRICE_",
+ "name":"MIN_PRICE_",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":19,
+ "scale":4,
+ "columnType":3,
+ "columnTypeName":"DECIMAL(19, 4)",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ },
+ {
+ "isNullable":1,
+ "displaySize":19,
+ "label":"MAX_PRICE_",
+ "name":"MAX_PRICE_",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":19,
+ "scale":4,
+ "columnType":3,
+ "columnTypeName":"DECIMAL(19, 4)",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ },
+ {
+ "isNullable":1,
+ "displaySize":19,
+ "label":"COUNT__",
+ "name":"COUNT__",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":19,
+ "scale":0,
+ "columnType":-5,
+ "columnTypeName":"BIGINT",
+ "readOnly":true,
+ "writable":false,
+ "caseSensitive":true,
+ "searchable":false,
+ "currency":false,
+ "signed":true,
+ "autoIncrement":false,
+ "definitelyWritable":false
+ }
+ ],
+ "results":[
+ [
+ "2013-08-07",
+ "32996",
+ "15",
+ "15",
+ "Auction",
+ "10000000",
+ "49.048952730908745",
+ "49.048952730908745",
+ "49.048952730908745",
+ "1"
+ ],
+ [
+ "2013-08-07",
+ "43398",
+ "0",
+ "14",
+ "ABIN",
+ "10000633",
+ "85.78317064220418",
+ "85.78317064220418",
+ "85.78317064220418",
+ "1"
+ ],
+ [
+ "2013-08-07",
+ "170083",
+ "3",
+ "12",
+ "FP-non GTC",
+ "10000688",
+ "49.4518822857833",
+ "49.4518822857833",
+ "49.4518822857833",
+ "1"
+ ],
+ [
+ "2013-08-07",
+ "73506",
+ "0",
+ "14",
+ "Others",
+ "10000858",
+ "35.7556514660872",
+ "35.7556514660872",
+ "35.7556514660872",
+ "1"
+ ],
+ [
+ "2013-08-07",
+ "26262",
+ "0",
+ "5",
+ "FP-GTC",
+ "10000914",
+ "69.63202173718798",
+ "69.63202173718798",
+ "69.63202173718798",
+ "1"
+ ],
+ [
+ "2013-08-07",
+ "1161",
+ "3",
+ "12",
+ "Others",
+ "10000959",
+ "76.72392839984632",
+ "76.72392839984632",
+ "76.72392839984632",
+ "1"
+ ]
+ ],
+ "cube":"test_kylin_cube_with_slr_desc",
+ "affectedRowCount":0,
+ "isException":false,
+ "exceptionMessage":null,
+ "duration":3451,
+ "partial":false
+}
+```
+
+***
+
+### List queryable tables
+`GET /tables_and_columns`
+#### Request Parameters
+* project - `required` `string` The project to load tables
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+[
+ {
+ "columns":[
+ {
+ "table_NAME":"TEST_CAL_DT",
+ "table_SCHEM":"EDW",
+ "column_NAME":"CAL_DT",
+ "data_TYPE":91,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":1,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"DATE"
+ },
+ {
+ "table_NAME":"TEST_CAL_DT",
+ "table_SCHEM":"EDW",
+ "column_NAME":"WEEK_BEG_DT",
+ "data_TYPE":91,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":2,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"DATE"
+ }
+ ],
+ "table_NAME":"TEST_CAL_DT",
+ "table_SCHEM":"EDW",
+ "ref_GENERATION":null,
+ "self_REFERENCING_COL_NAME":null,
+ "type_SCHEM":null,
+ "table_TYPE":"TABLE",
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_CAT":null,
+ "type_NAME":null
+ },
+ {
+ "columns":[
+ {
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "column_NAME":"LEAF_CATEG_ID",
+ "data_TYPE":4,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":1,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"INTEGER"
+ },
+ {
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "column_NAME":"SITE_ID",
+ "data_TYPE":4,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":2,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"INTEGER"
+ },
+ {
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "column_NAME":"META_CATEG_NAME",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":3,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ },
+ {
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "column_NAME":"CATEG_LVL2_NAME",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":4,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ },
+ {
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "column_NAME":"CATEG_LVL3_NAME",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":5,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ },
+ {
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "column_NAME":"USER_DEFINED_FIELD1",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":6,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ },
+ {
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "column_NAME":"USER_DEFINED_FIELD3",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":7,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ },
+ {
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "column_NAME":"UPD_DATE",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":8,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ },
+ {
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "column_NAME":"UPD_USER",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":9,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ }
+ ],
+ "table_NAME":"TEST_CATEGORY_GROUPINGS",
+ "table_SCHEM":"EDW",
+ "ref_GENERATION":null,
+ "self_REFERENCING_COL_NAME":null,
+ "type_SCHEM":null,
+ "table_TYPE":"TABLE",
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_CAT":null,
+ "type_NAME":null
+ },
+ {
+ "columns":[
+ {
+ "table_NAME":"TEST_KYLIN_FACT",
+ "table_SCHEM":"EDW",
+ "column_NAME":"CAL_DT",
+ "data_TYPE":91,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":1,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"DATE"
+ },
+ {
+ "table_NAME":"TEST_KYLIN_FACT",
+ "table_SCHEM":"EDW",
+ "column_NAME":"LEAF_CATEG_ID",
+ "data_TYPE":4,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":2,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"INTEGER"
+ },
+ {
+ "table_NAME":"TEST_KYLIN_FACT",
+ "table_SCHEM":"EDW",
+ "column_NAME":"LSTG_SITE_ID",
+ "data_TYPE":4,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":3,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"INTEGER"
+ },
+ {
+ "table_NAME":"TEST_KYLIN_FACT",
+ "table_SCHEM":"EDW",
+ "column_NAME":"SLR_SEGMENT_CD",
+ "data_TYPE":5,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":4,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"SMALLINT"
+ },
+ {
+ "table_NAME":"TEST_KYLIN_FACT",
+ "table_SCHEM":"EDW",
+ "column_NAME":"LSTG_FORMAT_NAME",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":5,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ },
+ {
+ "table_NAME":"TEST_KYLIN_FACT",
+ "table_SCHEM":"EDW",
+ "column_NAME":"SELLER_ID",
+ "data_TYPE":-5,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":6,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"BIGINT"
+ },
+ {
+ "table_NAME":"TEST_KYLIN_FACT",
+ "table_SCHEM":"EDW",
+ "column_NAME":"PRICE",
+ "data_TYPE":3,
+ "nullable":1,
+ "column_SIZE":19,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":4,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":19,
+ "ordinal_POSITION":7,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"DECIMAL(19, 4)"
+ }
+ ],
+ "table_NAME":"TEST_KYLIN_FACT",
+ "table_SCHEM":"EDW",
+ "ref_GENERATION":null,
+ "self_REFERENCING_COL_NAME":null,
+ "type_SCHEM":null,
+ "table_TYPE":"TABLE",
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_CAT":null,
+ "type_NAME":null
+ },
+ {
+ "columns":[
+ {
+ "table_NAME":"TEST_SELLER_TYPE_DIM",
+ "table_SCHEM":"EDW",
+ "column_NAME":"SELLER_TYPE_CD",
+ "data_TYPE":5,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":1,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"SMALLINT"
+ },
+ {
+ "table_NAME":"TEST_SELLER_TYPE_DIM",
+ "table_SCHEM":"EDW",
+ "column_NAME":"SELLER_TYPE_DESC",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":2,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ }
+ ],
+ "table_NAME":"TEST_SELLER_TYPE_DIM",
+ "table_SCHEM":"EDW",
+ "ref_GENERATION":null,
+ "self_REFERENCING_COL_NAME":null,
+ "type_SCHEM":null,
+ "table_TYPE":"TABLE",
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_CAT":null,
+ "type_NAME":null
+ },
+ {
+ "columns":[
+ {
+ "table_NAME":"TEST_SITES",
+ "table_SCHEM":"EDW",
+ "column_NAME":"SITE_ID",
+ "data_TYPE":4,
+ "nullable":1,
+ "column_SIZE":-1,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":-1,
+ "ordinal_POSITION":1,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"INTEGER"
+ },
+ {
+ "table_NAME":"TEST_SITES",
+ "table_SCHEM":"EDW",
+ "column_NAME":"SITE_NAME",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":2,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ },
+ {
+ "table_NAME":"TEST_SITES",
+ "table_SCHEM":"EDW",
+ "column_NAME":"CRE_USER",
+ "data_TYPE":12,
+ "nullable":1,
+ "column_SIZE":256,
+ "buffer_LENGTH":-1,
+ "decimal_DIGITS":0,
+ "num_PREC_RADIX":10,
+ "column_DEF":null,
+ "sql_DATA_TYPE":-1,
+ "sql_DATETIME_SUB":-1,
+ "char_OCTET_LENGTH":256,
+ "ordinal_POSITION":3,
+ "is_NULLABLE":"YES",
+ "scope_CATLOG":null,
+ "scope_SCHEMA":null,
+ "scope_TABLE":null,
+ "source_DATA_TYPE":-1,
+ "iS_AUTOINCREMENT":null,
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_NAME":"VARCHAR(256) CHARACTER SET \"UTF-16LE\" COLLATE \"UTF-16LE$en_US$primary\""
+ }
+ ],
+ "table_NAME":"TEST_SITES",
+ "table_SCHEM":"EDW",
+ "ref_GENERATION":null,
+ "self_REFERENCING_COL_NAME":null,
+ "type_SCHEM":null,
+ "table_TYPE":"TABLE",
+ "table_CAT":"defaultCatalog",
+ "remarks":null,
+ "type_CAT":null,
+ "type_NAME":null
+ }
+]
+```
+
+
+***
+
+### List cubes
+`GET /cubes`
+#### Request Parameters
+* cubeName - `optional` `string` Cube name to find.
+* offset - `required` `int` Offset used by pagination
+* limit - `required` `int ` Cubes per page.
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+[
+ {
+ "uuid":"1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
+ "last_modified":1407831634847,
+ "name":"test_kylin_cube_with_slr_empty",
+ "owner":null,
+ "version":null,
+ "descriptor":"test_kylin_cube_with_slr_desc",
+ "cost":50,
+ "status":"DISABLED",
+ "segments":[
+ ],
+ "create_time":null,
+ "source_records_count":0,
+ "source_records_size":0,
+ "size_kb":0
+ }
+]
+```
+
+### Rebuild cube
+`PUT /cubes/{cubeName}/rebuild`
+#### Path Variable
+* cubeName - `required` `string` Cube name.
+
+#### Request Body
+* startTime - `required` `long` Start timestamp of data to build, e.g. 1388563200000 for 2014-1-1
+* endTime - `required` `long` End timestamp of data to build
+* buildType - `required` `string` Build type: 'BUILD' OR 'MERGE'
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ "uuid":"c143e0e4-ac5f-434d-acf3-46b0d15e3dc6",
+ "last_modified":1407908916705,
+ "name":"test_kylin_cube_with_slr_empty - 19700101000000_20140731160000 - BUILD - PDT 2014-08-12 22:48:36",
+ "type":"BUILD",
+ "duration":0,
+ "related_cube":"test_kylin_cube_with_slr_empty",
+ "related_segment":"19700101000000_20140731160000",
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "mr_waiting":0,
+ "steps":[
+ {
+ "interruptCmd":null,
+ "name":"Create Intermediate Flat Hive Table",
+ "sequence_id":0,
+ "exec_cmd":"hive -e \"DROP TABLE IF EXISTS kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6;\nCREATE EXTERNAL TABLE IF NOT EXISTS kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6\n(\nCAL_DT date\n,LEAF_CATEG_ID int\n,LSTG_SITE_ID int\n,META_CATEG_NAME string\n,CATEG_LVL2_NAME string\n,CATEG_LVL3_NAME string\n,LSTG_FORMAT_NAME string\n,SLR_SEGMENT_CD smallint\n,SELLER_ID bigint\n,PRICE decimal\n)\nROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'\nSTORED AS SEQUENCEFILE\nLOCATION '/tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6';\nSET mapreduce.job.split.metainfo.maxsize=-1;\nSET mapred.compress.map.output=true;\nSET mapred.map.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;\nSET mapred.output.compress=true;\nSET ma
pred.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;\nSET mapred.output.compression.type=BLOCK;\nSET mapreduce.job.max.split.locations=2000;\nSET hive.exec.compress.output=true;\nSET hive.auto.convert.join.noconditionaltask = true;\nSET hive.auto.convert.join.noconditionaltask.size = 300000000;\nINSERT OVERWRITE TABLE kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6\nSELECT\nTEST_KYLIN_FACT.CAL_DT\n,TEST_KYLIN_FACT.LEAF_CATEG_ID\n,TEST_KYLIN_FACT.LSTG_SITE_ID\n,TEST_CATEGORY_GROUPINGS.META_CATEG_NAME\n,TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME\n,TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME\n,TEST_KYLIN_FACT.LSTG_FORMAT_NAME\n,TEST_KYLIN_FACT.SLR_SEGMENT_CD\n,TEST_KYLIN_FACT.SELLER_ID\n,TEST_KYLIN_FACT.PRICE\nFROM TEST_KYLIN_FACT\nINNER JOIN TEST_CAL_DT\nON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT\nINNER JOIN TEST_CATEGORY_GROUPINGS\nON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AN
D TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID\nINNER JOIN TEST_SITES\nON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID\nINNER JOIN TEST_SELLER_TYPE_DIM\nON TEST_KYLIN_FACT.SLR_SEGMENT_CD = TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD\nWHERE (test_kylin_fact.cal_dt < '2014-07-31 16:00:00')\n;\n\"",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"SHELL_CMD_HADOOP",
+ "info":null,
+ "run_async":false
+ },
+ {
+ "interruptCmd":null,
+ "name":"Extract Fact Table Distinct Columns",
+ "sequence_id":1,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6 -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/fact_distinct_columns -jobname Kylin_Fact_Distinct_Columns_test_kylin_cube_with_slr_empty_Step_1",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_FACTDISTINCT",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build Dimension Dictionary",
+ "sequence_id":2,
+ "exec_cmd":" -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/fact_distinct_columns",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NO_MR_DICTIONARY",
+ "info":null,
+ "run_async":false
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build Base Cuboid Data",
+ "sequence_id":3,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6 -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/base_cuboid -jobname Kylin_Base_Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_3 -level 0",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_BASECUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 8-Dimension",
+ "sequence_id":4,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/base_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/8d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_4 -level 1",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 7-Dimension",
+ "sequence_id":5,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/8d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/7d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_5 -level 2",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 6-Dimension",
+ "sequence_id":6,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/7d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/6d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_6 -level 3",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 5-Dimension",
+ "sequence_id":7,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/6d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/5d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_7 -level 4",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 4-Dimension",
+ "sequence_id":8,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/5d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/4d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_8 -level 5",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Calculate HTable Region Splits",
+ "sequence_id":9,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/* -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/rowkey_stats -jobname Kylin_Region_Splits_Calculator_test_kylin_cube_with_slr_empty_Step_9 -cubename test_kylin_cube_with_slr_empty",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_RANGEKEYDISTRIBUTION",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Create HTable",
+ "sequence_id":10,
+ "exec_cmd":" -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/rowkey_stats/part-r-00000 -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADDOP_NO_MR_CREATEHTABLE",
+ "info":null,
+ "run_async":false
+ },
+ {
+ "interruptCmd":null,
+ "name":"Convert Cuboid Data to HFile",
+ "sequence_id":11,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/* -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/hfile -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A -jobname Kylin_HFile_Generator_test_kylin_cube_with_slr_empty_Step_11",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_CONVERTHFILE",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Load HFile to HBase Table",
+ "sequence_id":12,
+ "exec_cmd":" -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/hfile/ -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A -cubename test_kylin_cube_with_slr_empty",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NO_MR_BULKLOAD",
+ "info":null,
+ "run_async":false
+ }
+ ],
+ "job_status":"PENDING",
+ "progress":0.0
+}
+```
+
+
+### Disable Cube
+`PUT /cubes/{cubeName}/disable`
+#### Path variable
+* cubeName - `required` `string` Cube name.
+
+#### Response Sample
+(Same as "Enable Cube")
+
+
+### Purge Cube
+`PUT /cubes/{cubeName}/purge`
+#### Path variable
+* cubeName - `required` `string` Cube name.
+
+#### Response Sample
+(Same as "Enable Cube")
+
+
+### Enable Cube
+`PUT /cubes/{cubeName}/enable`
+#### Path variable
+* cubeName - `required` `string` Cube name.
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ "uuid":"1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
+ "last_modified":1407909046305,
+ "name":"test_kylin_cube_with_slr_ready",
+ "owner":null,
+ "version":null,
+ "descriptor":"test_kylin_cube_with_slr_desc",
+ "cost":50,
+ "status":"ACTIVE",
+ "segments":[
+ {
+ "name":"19700101000000_20140531160000",
+ "storage_location_identifier":"KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-19700101000000_20140531160000_BF043D2D-9A4A-45E9-AA59-5A17D3F34A50",
+ "date_range_start":0,
+ "date_range_end":1401552000000,
+ "status":"READY",
+ "size_kb":4758,
+ "source_records":6000,
+ "source_records_size":620356,
+ "last_build_time":1407832663227,
+ "last_build_job_id":"2c7a2b63-b052-4a51-8b09-0c24b5792cda",
+ "binary_signature":null,
+ "dictionaries":{
+ "TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME":"/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
+ "TEST_KYLIN_FACT/LSTG_SITE_ID":"/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict",
+ "TEST_KYLIN_FACT/SLR_SEGMENT_CD":"/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict",
+ "TEST_KYLIN_FACT/CAL_DT":"/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
+ "TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME":"/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/270fbfb0-281c-4602-8413-2970a7439c47.dict",
+ "TEST_KYLIN_FACT/LEAF_CATEG_ID":"/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict",
+ "TEST_CATEGORY_GROUPINGS/META_CATEG_NAME":"/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict"
+ },
+ "snapshots":{
+ "TEST_CAL_DT":"/table_snapshot/TEST_CAL_DT.csv/8f7cfc8a-020d-4019-b419-3c6deb0ffaa0.snapshot",
+ "TEST_SELLER_TYPE_DIM":"/table_snapshot/TEST_SELLER_TYPE_DIM.csv/c60fd05e-ac94-4016-9255-96521b273b81.snapshot",
+ "TEST_CATEGORY_GROUPINGS":"/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/363f4a59-b725-4459-826d-3188bde6a971.snapshot",
+ "TEST_SITES":"/table_snapshot/TEST_SITES.csv/78e0aecc-3ec6-4406-b86e-bac4b10ea63b.snapshot"
+ }
+ }
+ ],
+ "create_time":null,
+ "source_records_count":6000,
+ "source_records_size":0,
+ "size_kb":4758
+}
+```
+
+### Resume Job
+`PUT /jobs/{jobId}/resume`
+
+#### Path variable
+* jobId- `required` `string` Job id.
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ "uuid":"c143e0e4-ac5f-434d-acf3-46b0d15e3dc6",
+ "last_modified":1407908916705,
+ "name":"test_kylin_cube_with_slr_empty - 19700101000000_20140731160000 - BUILD - PDT 2014-08-12 22:48:36",
+ "type":"BUILD",
+ "duration":0,
+ "related_cube":"test_kylin_cube_with_slr_empty",
+ "related_segment":"19700101000000_20140731160000",
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "mr_waiting":0,
+ "steps":[
+ {
+ "interruptCmd":null,
+ "name":"Create Intermediate Flat Hive Table",
+ "sequence_id":0,
+ "exec_cmd":"hive -e \"DROP TABLE IF EXISTS kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6;\nCREATE EXTERNAL TABLE IF NOT EXISTS kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6\n(\nCAL_DT date\n,LEAF_CATEG_ID int\n,LSTG_SITE_ID int\n,META_CATEG_NAME string\n,CATEG_LVL2_NAME string\n,CATEG_LVL3_NAME string\n,LSTG_FORMAT_NAME string\n,SLR_SEGMENT_CD smallint\n,SELLER_ID bigint\n,PRICE decimal\n)\nROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'\nSTORED AS SEQUENCEFILE\nLOCATION '/tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6';\nSET mapreduce.job.split.metainfo.maxsize=-1;\nSET mapred.compress.map.output=true;\nSET mapred.map.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;\nSET mapred.output.compress=true;\nSET ma
pred.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;\nSET mapred.output.compression.type=BLOCK;\nSET mapreduce.job.max.split.locations=2000;\nSET hive.exec.compress.output=true;\nSET hive.auto.convert.join.noconditionaltask = true;\nSET hive.auto.convert.join.noconditionaltask.size = 300000000;\nINSERT OVERWRITE TABLE kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6\nSELECT\nTEST_KYLIN_FACT.CAL_DT\n,TEST_KYLIN_FACT.LEAF_CATEG_ID\n,TEST_KYLIN_FACT.LSTG_SITE_ID\n,TEST_CATEGORY_GROUPINGS.META_CATEG_NAME\n,TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME\n,TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME\n,TEST_KYLIN_FACT.LSTG_FORMAT_NAME\n,TEST_KYLIN_FACT.SLR_SEGMENT_CD\n,TEST_KYLIN_FACT.SELLER_ID\n,TEST_KYLIN_FACT.PRICE\nFROM TEST_KYLIN_FACT\nINNER JOIN TEST_CAL_DT\nON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT\nINNER JOIN TEST_CATEGORY_GROUPINGS\nON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AN
D TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID\nINNER JOIN TEST_SITES\nON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID\nINNER JOIN TEST_SELLER_TYPE_DIM\nON TEST_KYLIN_FACT.SLR_SEGMENT_CD = TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD\nWHERE (test_kylin_fact.cal_dt < '2014-07-31 16:00:00')\n;\n\"",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"SHELL_CMD_HADOOP",
+ "info":null,
+ "run_async":false
+ },
+ {
+ "interruptCmd":null,
+ "name":"Extract Fact Table Distinct Columns",
+ "sequence_id":1,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6 -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/fact_distinct_columns -jobname Kylin_Fact_Distinct_Columns_test_kylin_cube_with_slr_empty_Step_1",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_FACTDISTINCT",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build Dimension Dictionary",
+ "sequence_id":2,
+ "exec_cmd":" -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/fact_distinct_columns",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NO_MR_DICTIONARY",
+ "info":null,
+ "run_async":false
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build Base Cuboid Data",
+ "sequence_id":3,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6 -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/base_cuboid -jobname Kylin_Base_Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_3 -level 0",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_BASECUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 8-Dimension",
+ "sequence_id":4,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/base_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/8d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_4 -level 1",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 7-Dimension",
+ "sequence_id":5,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/8d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/7d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_5 -level 2",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 6-Dimension",
+ "sequence_id":6,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/7d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/6d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_6 -level 3",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 5-Dimension",
+ "sequence_id":7,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/6d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/5d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_7 -level 4",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 4-Dimension",
+ "sequence_id":8,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/5d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/4d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_8 -level 5",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Calculate HTable Region Splits",
+ "sequence_id":9,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/* -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/rowkey_stats -jobname Kylin_Region_Splits_Calculator_test_kylin_cube_with_slr_empty_Step_9 -cubename test_kylin_cube_with_slr_empty",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_RANGEKEYDISTRIBUTION",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Create HTable",
+ "sequence_id":10,
+ "exec_cmd":" -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/rowkey_stats/part-r-00000 -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADDOP_NO_MR_CREATEHTABLE",
+ "info":null,
+ "run_async":false
+ },
+ {
+ "interruptCmd":null,
+ "name":"Convert Cuboid Data to HFile",
+ "sequence_id":11,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/* -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/hfile -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A -jobname Kylin_HFile_Generator_test_kylin_cube_with_slr_empty_Step_11",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_CONVERTHFILE",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Load HFile to HBase Table",
+ "sequence_id":12,
+ "exec_cmd":" -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/hfile/ -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A -cubename test_kylin_cube_with_slr_empty",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NO_MR_BULKLOAD",
+ "info":null,
+ "run_async":false
+ }
+ ],
+ "job_status":"PENDING",
+ "progress":0.0
+}
+```
+
+### Discard Job
+`PUT /jobs/{jobId}/cancel`
+#### Path variable
+* jobId- `required` `string` Job id.
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ "uuid":"c143e0e4-ac5f-434d-acf3-46b0d15e3dc6",
+ "last_modified":1407908916705,
+ "name":"test_kylin_cube_with_slr_empty - 19700101000000_20140731160000 - BUILD - PDT 2014-08-12 22:48:36",
+ "type":"BUILD",
+ "duration":0,
+ "related_cube":"test_kylin_cube_with_slr_empty",
+ "related_segment":"19700101000000_20140731160000",
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "mr_waiting":0,
+ "steps":[
+ {
+ "interruptCmd":null,
+ "name":"Create Intermediate Flat Hive Table",
+ "sequence_id":0,
+ "exec_cmd":"hive -e \"DROP TABLE IF EXISTS kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6;\nCREATE EXTERNAL TABLE IF NOT EXISTS kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6\n(\nCAL_DT date\n,LEAF_CATEG_ID int\n,LSTG_SITE_ID int\n,META_CATEG_NAME string\n,CATEG_LVL2_NAME string\n,CATEG_LVL3_NAME string\n,LSTG_FORMAT_NAME string\n,SLR_SEGMENT_CD smallint\n,SELLER_ID bigint\n,PRICE decimal\n)\nROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'\nSTORED AS SEQUENCEFILE\nLOCATION '/tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6';\nSET mapreduce.job.split.metainfo.maxsize=-1;\nSET mapred.compress.map.output=true;\nSET mapred.map.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;\nSET mapred.output.compress=true;\nSET ma
pred.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;\nSET mapred.output.compression.type=BLOCK;\nSET mapreduce.job.max.split.locations=2000;\nSET hive.exec.compress.output=true;\nSET hive.auto.convert.join.noconditionaltask = true;\nSET hive.auto.convert.join.noconditionaltask.size = 300000000;\nINSERT OVERWRITE TABLE kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6\nSELECT\nTEST_KYLIN_FACT.CAL_DT\n,TEST_KYLIN_FACT.LEAF_CATEG_ID\n,TEST_KYLIN_FACT.LSTG_SITE_ID\n,TEST_CATEGORY_GROUPINGS.META_CATEG_NAME\n,TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME\n,TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME\n,TEST_KYLIN_FACT.LSTG_FORMAT_NAME\n,TEST_KYLIN_FACT.SLR_SEGMENT_CD\n,TEST_KYLIN_FACT.SELLER_ID\n,TEST_KYLIN_FACT.PRICE\nFROM TEST_KYLIN_FACT\nINNER JOIN TEST_CAL_DT\nON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT\nINNER JOIN TEST_CATEGORY_GROUPINGS\nON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AN
D TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID\nINNER JOIN TEST_SITES\nON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID\nINNER JOIN TEST_SELLER_TYPE_DIM\nON TEST_KYLIN_FACT.SLR_SEGMENT_CD = TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD\nWHERE (test_kylin_fact.cal_dt < '2014-07-31 16:00:00')\n;\n\"",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"SHELL_CMD_HADOOP",
+ "info":null,
+ "run_async":false
+ },
+ {
+ "interruptCmd":null,
+ "name":"Extract Fact Table Distinct Columns",
+ "sequence_id":1,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6 -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/fact_distinct_columns -jobname Kylin_Fact_Distinct_Columns_test_kylin_cube_with_slr_empty_Step_1",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_FACTDISTINCT",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build Dimension Dictionary",
+ "sequence_id":2,
+ "exec_cmd":" -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/fact_distinct_columns",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NO_MR_DICTIONARY",
+ "info":null,
+ "run_async":false
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build Base Cuboid Data",
+ "sequence_id":3,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6 -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/base_cuboid -jobname Kylin_Base_Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_3 -level 0",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_BASECUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 8-Dimension",
+ "sequence_id":4,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/base_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/8d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_4 -level 1",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 7-Dimension",
+ "sequence_id":5,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/8d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/7d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_5 -level 2",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 6-Dimension",
+ "sequence_id":6,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/7d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/6d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_6 -level 3",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 5-Dimension",
+ "sequence_id":7,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/6d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/5d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_7 -level 4",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Build N-Dimension Cuboid Data : 4-Dimension",
+ "sequence_id":8,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -segmentname 19700101000000_20140731160000 -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/5d_cuboid -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/4d_cuboid -jobname Kylin_ND-Cuboid_Builder_test_kylin_cube_with_slr_empty_Step_8 -level 5",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NDCUBOID",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Calculate HTable Region Splits",
+ "sequence_id":9,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/* -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/rowkey_stats -jobname Kylin_Region_Splits_Calculator_test_kylin_cube_with_slr_empty_Step_9 -cubename test_kylin_cube_with_slr_empty",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_RANGEKEYDISTRIBUTION",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Create HTable",
+ "sequence_id":10,
+ "exec_cmd":" -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/rowkey_stats/part-r-00000 -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADDOP_NO_MR_CREATEHTABLE",
+ "info":null,
+ "run_async":false
+ },
+ {
+ "interruptCmd":null,
+ "name":"Convert Cuboid Data to HFile",
+ "sequence_id":11,
+ "exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/cuboid/* -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/hfile -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A -jobname Kylin_HFile_Generator_test_kylin_cube_with_slr_empty_Step_11",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_CONVERTHFILE",
+ "info":null,
+ "run_async":true
+ },
+ {
+ "interruptCmd":null,
+ "name":"Load HFile to HBase Table",
+ "sequence_id":12,
+ "exec_cmd":" -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/hfile/ -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A -cubename test_kylin_cube_with_slr_empty",
+ "interrupt_cmd":null,
+ "exec_start_time":0,
+ "exec_end_time":0,
+ "exec_wait_time":0,
+ "step_status":"PENDING",
+ "cmd_type":"JAVA_CMD_HADOOP_NO_MR_BULKLOAD",
+ "info":null,
+ "run_async":false
+ }
+ ],
+ "job_status":"PENDING",
+ "progress":0.0
+}
+```
+
+### Get job step output
+`GET /{jobId}/steps/{stepId}/output`
+#### Path Variable
+* jobId - `required` `string` Job id.
+* stepId - `required` `string` Step id; the step id is composed by jobId with step sequence id; for example, the jobId is "fb479e54-837f-49a2-b457-651fc50be110", its 3rd step id is "fb479e54-837f-49a2-b457-651fc50be110-3",
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ "cmd_output":"log string"
+}
+```
+
+
+### Get Hive Table
+`GET /tables/{tableName}`
+#### Request Parameters
+* tableName - `required` `string` table name to find.
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ uuid: "69cc92c0-fc42-4bb9-893f-bd1141c91dbe",
+ name: "SAMPLE_07",
+ columns: [{
+ id: "1",
+ name: "CODE",
+ datatype: "string"
+ }, {
+ id: "2",
+ name: "DESCRIPTION",
+ datatype: "string"
+ }, {
+ id: "3",
+ name: "TOTAL_EMP",
+ datatype: "int"
+ }, {
+ id: "4",
+ name: "SALARY",
+ datatype: "int"
+ }],
+ database: "DEFAULT",
+ last_modified: 1419330476755
+}
+```
+
+### Get Hive Table (Extend Info)
+`GET /tables/{tableName}/exd-map`
+#### Request Parameters
+* tableName - `optional` `string` table name to find.
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ "minFileSize": "46055",
+ "totalNumberFiles": "1",
+ "location": "hdfs://sandbox.hortonworks.com:8020/apps/hive/warehouse/sample_07",
+ "lastAccessTime": "1418374103365",
+ "lastUpdateTime": "1398176493340",
+ "columns": "struct columns { string code, string description, i32 total_emp, i32 salary}",
+ "partitionColumns": "",
+ "EXD_STATUS": "true",
+ "maxFileSize": "46055",
+ "inputformat": "org.apache.hadoop.mapred.TextInputFormat",
+ "partitioned": "false",
+ "tableName": "sample_07",
+ "owner": "hue",
+ "totalFileSize": "46055",
+ "outputformat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+}
+```
+
+### Get Hive Tables
+`GET /tables`
+#### Request Parameters
+* project- `required` `string` will list all tables in the project.
+* ext- `optional` `boolean` set true to get extend info of table.
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+[
+ {
+ uuid: "53856c96-fe4d-459e-a9dc-c339b1bc3310",
+ name: "SAMPLE_08",
+ columns: [{
+ id: "1",
+ name: "CODE",
+ datatype: "string"
+ }, {
+ id: "2",
+ name: "DESCRIPTION",
+ datatype: "string"
+ }, {
+ id: "3",
+ name: "TOTAL_EMP",
+ datatype: "int"
+ }, {
+ id: "4",
+ name: "SALARY",
+ datatype: "int"
+ }],
+ database: "DEFAULT",
+ cardinality: {},
+ last_modified: 0,
+ exd: {
+ minFileSize: "46069",
+ totalNumberFiles: "1",
+ location: "hdfs://sandbox.hortonworks.com:8020/apps/hive/warehouse/sample_08",
+ lastAccessTime: "1398176495945",
+ lastUpdateTime: "1398176495981",
+ columns: "struct columns { string code, string description, i32 total_emp, i32 salary}",
+ partitionColumns: "",
+ EXD_STATUS: "true",
+ maxFileSize: "46069",
+ inputformat: "org.apache.hadoop.mapred.TextInputFormat",
+ partitioned: "false",
+ tableName: "sample_08",
+ owner: "hue",
+ totalFileSize: "46069",
+ outputformat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+ }
+}, {
+ uuid: "82d72d9c-705c-4706-aaf5-358ac2661cf9",
+ name: "TEST_KYLIN_FACT",
+ columns: [{
+ id: "1",
+ name: "TRANS_ID",
+ datatype: "bigint"
+ }, {
+ id: "2",
+ name: "CAL_DT",
+ datatype: "date"
+ }, {
+ id: "3",
+ name: "LSTG_FORMAT_NAME",
+ datatype: "string"
+ }, {
+ id: "4",
+ name: "LEAF_CATEG_ID",
+ datatype: "int"
+ }, {
+ id: "5",
+ name: "LSTG_SITE_ID",
+ datatype: "int"
+ }, {
+ id: "6",
+ name: "SLR_SEGMENT_CD",
+ datatype: "smallint"
+ }, {
+ id: "7",
+ name: "PRICE",
+ datatype: "decimal(38,16)"
+ }, {
+ id: "8",
+ name: "ITEM_COUNT",
+ datatype: "bigint"
+ }, {
+ id: "9",
+ name: "SELLER_ID",
+ datatype: "bigint"
+ }],
+ database: "DEFAULT",
+ cardinality: {},
+ last_modified: 0,
+ exd: {
+ minFileSize: "607328",
+ totalNumberFiles: "1",
+ location: "hdfs://sandbox.hortonworks.com:8020/apps/hive/warehouse/test_kylin_fact",
+ lastAccessTime: "1418899958520",
+ lastUpdateTime: "1414051037179",
+ columns: "struct columns { i64 trans_id, date cal_dt, string lstg_format_name, i32 leaf_categ_id, i32 lstg_site_id, i16 slr_segment_cd, decimal(38,16) price, i64 item_count, i64 seller_id}",
+ partitionColumns: "",
+ EXD_STATUS: "true",
+ maxFileSize: "607328",
+ inputformat: "org.apache.hadoop.mapred.TextInputFormat",
+ partitioned: "false",
+ tableName: "test_kylin_fact",
+ owner: "root",
+ totalFileSize: "607328",
+ outputformat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+ }
+ }
+]
+```
+
+### Load Hive Tables
+`POST /tables/{tables}/{project}`
+
+* tables- `required` `string` tables you want to load from hive,split with comma.
+* project- `required` `String` specif.
+
+
+#### Response Sample
+```sh
+Status: 200OK
+```
+```sh
+{
+ "result.loaded": ["DEFAULT.SAMPLE_07"],
+ "result.unloaded": ["sapmle_08"]
+}
+```
+
+
+### Wipe cache
+`GET /cache/{type}/{name}/{action}`
+#### Path variable
+* type - `required` `string` 'METADATA' or 'CUBE'
+* name - `required` `string` Cache key.
+* action - `required` `string` 'create', 'update' or 'drop'
+
+#### Response Sample
+```sh
+Status: 200OK
+```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5acdcd23/docs/TechInside/New meta data model structure.md
----------------------------------------------------------------------
diff --git a/docs/TechInside/New meta data model structure.md b/docs/TechInside/New meta data model structure.md
new file mode 100644
index 0000000..74fc74b
--- /dev/null
+++ b/docs/TechInside/New meta data model structure.md
@@ -0,0 +1,18 @@
+#New meta data model structure
+Kylin is doing a round code refactor which will introduce the following two changes on meta data:
+
+* Abstract a "model_desc" layer from "cube_desc"
+
+Before define a cube, user will need firstly define a model ("model_desc"); The model defines which is the fact table, which are lookup tables and how they be joined;
+
+With the model be defined, when user define a cube ("cube_desc"), he/she only need to specify the table/column name for a dimension, as the join conditions have already been defined;
+
+This abstraction is to extend the meta data to fulfill non-cube queries (coming soon);
+
+* Support data tables from multiple hive databases;
+
+User has the case that tables are from multiple hive database, and the table name might be the same; To support this case Kylin will use the database name + table name as the unique name for tables; And user need to specify the database name (if it is not "default") in SQL when query Kylin.
+
+Here is a sample; the fact table "test_kylin_fact" is from default hive database, you don't need to specify the db name; while lookup table is from "edw", you need use "edw.test_cal_dt" in the query:
+
+ select test_cal_dt.Week_Beg_Dt, sum(price) as c1, count(1) as c2 from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt where test_kylin_fact.lstg_format_name='ABIN'
\ No newline at end of file