You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:14 UTC
[01/50] incubator-kylin git commit: Update storageTest
Repository: incubator-kylin
Updated Branches:
refs/heads/streaming 825409504 -> 624c4347f
Update storageTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d7cf8a76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d7cf8a76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d7cf8a76
Branch: refs/heads/streaming
Commit: d7cf8a7667f8144a8483bc9a68af691586a52619
Parents: 20b1b37
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Mar 6 13:14:06 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Mar 6 13:14:06 2015 +0800
----------------------------------------------------------------------
.../src/test/java/org/apache/kylin/storage/test/StorageTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d7cf8a76/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
index 055001d..0280e85 100644
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
@@ -221,7 +221,7 @@ public class StorageTest extends HBaseMetadataTestCase {
CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
compareFilter.addChild(columnFilter1);
- ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2011-03-10");
+ ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2012-05-23");
compareFilter.addChild(constantFilter1);
return compareFilter;
}
[17/50] incubator-kylin git commit: KYLIN-625, split IEvaluatableTuple
Posted by li...@apache.org.
KYLIN-625, split IEvaluatableTuple
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fd0fbd5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fd0fbd5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fd0fbd5c
Branch: refs/heads/streaming
Commit: fd0fbd5cba7b1b97bac333b0ef9cd3bc77a5f04c
Parents: 9ec330a
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Mar 9 15:04:52 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Mar 9 15:04:52 2015 +0800
----------------------------------------------------------------------
.../kylin/metadata/filter/CaseTupleFilter.java | 4 +--
.../metadata/filter/ColumnTupleFilter.java | 3 +-
.../metadata/filter/CompareTupleFilter.java | 3 +-
.../metadata/filter/ConstantTupleFilter.java | 3 +-
.../metadata/filter/DynamicTupleFilter.java | 3 +-
.../metadata/filter/ExtractTupleFilter.java | 4 +--
.../metadata/filter/IEvaluatableTuple.java | 9 ++++++
.../metadata/filter/LogicalTupleFilter.java | 10 +++---
.../kylin/metadata/filter/TupleFilter.java | 6 ++--
.../org/apache/kylin/metadata/tuple/ITuple.java | 5 +--
.../storage/hbase/DerivedFilterTranslator.java | 32 ++++---------------
.../hbase/coprocessor/CoprocessorFilter.java | 14 ++++++---
.../coprocessor/observer/ObserverTuple.java | 33 ++------------------
13 files changed, 43 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
index 1b1d979..5acbe5c 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
@@ -18,8 +18,6 @@
package org.apache.kylin.metadata.filter;
-import org.apache.kylin.metadata.tuple.ITuple;
-
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -64,7 +62,7 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(ITuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple) {
if (whenFilters.size() != thenFilters.size()) {
elseFilter = whenFilters.remove(whenFilters.size() - 1);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index 865dab3..1cf1f3c 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -28,7 +28,6 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
/**
*
@@ -67,7 +66,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(ITuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple) {
this.tupleValue = tuple.getValue(columnRef);
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 6e1cca9..16114c5 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -27,7 +27,6 @@ import java.util.Map;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
/**
* @author xjiang
@@ -142,7 +141,7 @@ public class CompareTupleFilter extends TupleFilter {
// TODO requires generalize, currently only evaluates COLUMN {op} CONST
@Override
- public boolean evaluate(ITuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple) {
// extract tuple value
String tupleValue = null;
for (TupleFilter filter : this.children) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index 6234db2..1b21728 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -24,7 +24,6 @@ import java.util.Collections;
import java.util.HashSet;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.tuple.ITuple;
/**
*
@@ -64,7 +63,7 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(ITuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple) {
return constantValues.size() > 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
index c19a30f..08426ba 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
@@ -23,7 +23,6 @@ import java.util.Collection;
import java.util.Collections;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.tuple.ITuple;
/**
*
@@ -54,7 +53,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(ITuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple) {
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
index d96da2e..a02f91a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
@@ -18,8 +18,6 @@
package org.apache.kylin.metadata.filter;
-import org.apache.kylin.metadata.tuple.ITuple;
-
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -54,7 +52,7 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(ITuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple) {
// extract tuple value
String extractType = null;
String tupleValue = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/filter/IEvaluatableTuple.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/IEvaluatableTuple.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/IEvaluatableTuple.java
new file mode 100644
index 0000000..92f83f1
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/IEvaluatableTuple.java
@@ -0,0 +1,9 @@
+package org.apache.kylin.metadata.filter;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+public interface IEvaluatableTuple {
+
+ public Object getValue(TblColRef col);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 5d4f642..2649674 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -18,8 +18,6 @@
package org.apache.kylin.metadata.filter;
-import org.apache.kylin.metadata.tuple.ITuple;
-
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -71,7 +69,7 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(ITuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple) {
switch (this.operator) {
case AND:
return evalAnd(tuple);
@@ -84,7 +82,7 @@ public class LogicalTupleFilter extends TupleFilter {
}
}
- private boolean evalAnd(ITuple tuple) {
+ private boolean evalAnd(IEvaluatableTuple tuple) {
for (TupleFilter filter : this.children) {
if (!filter.evaluate(tuple)) {
return false;
@@ -93,7 +91,7 @@ public class LogicalTupleFilter extends TupleFilter {
return true;
}
- private boolean evalOr(ITuple tuple) {
+ private boolean evalOr(IEvaluatableTuple tuple) {
for (TupleFilter filter : this.children) {
if (filter.evaluate(tuple)) {
return true;
@@ -102,7 +100,7 @@ public class LogicalTupleFilter extends TupleFilter {
return false;
}
- private boolean evalNot(ITuple tuple) {
+ private boolean evalNot(IEvaluatableTuple tuple) {
return !this.children.get(0).evaluate(tuple);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index d3e6873..9a461c4 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -24,9 +24,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.common.collect.Maps;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
+
+import com.google.common.collect.Maps;
/**
*
@@ -188,7 +188,7 @@ public abstract class TupleFilter {
public abstract boolean isEvaluable();
- public abstract boolean evaluate(ITuple tuple);
+ public abstract boolean evaluate(IEvaluatableTuple tuple);
public abstract Collection<String> getValues();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java
index c1f6536..e5dbf08 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITuple.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.tuple;
import java.util.List;
+import org.apache.kylin.metadata.filter.IEvaluatableTuple;
import org.apache.kylin.metadata.model.TblColRef;
/**
@@ -28,7 +29,7 @@ import org.apache.kylin.metadata.model.TblColRef;
*
* @author yangli9
*/
-public interface ITuple {
+public interface ITuple extends IEvaluatableTuple {
public List<String> getAllFields();
@@ -36,7 +37,7 @@ public interface ITuple {
public Object[] getAllValues();
- public Object getValue(TblColRef col);
+ // declared in IEvaluatableTuple: public Object getValue(TblColRef col);
public Object getValue(String field);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
index e542555..bfa9d3c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
@@ -22,22 +22,22 @@ import java.util.List;
import java.util.Set;
import org.apache.hadoop.hbase.util.Pair;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.cube.kv.RowKeyColumnOrder;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.CubeDesc.DeriveType;
import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.IEvaluatableTuple;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* @author yangli9
@@ -172,7 +172,7 @@ public class DerivedFilterTranslator {
}
}
- private static class SingleColumnTuple implements ITuple {
+ private static class SingleColumnTuple implements IEvaluatableTuple {
private TblColRef col;
private String value;
@@ -182,21 +182,6 @@ public class DerivedFilterTranslator {
}
@Override
- public List<String> getAllFields() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<TblColRef> getAllColumns() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object[] getAllValues() {
- throw new UnsupportedOperationException();
- }
-
- @Override
public Object getValue(TblColRef col) {
if (this.col.equals(col))
return value;
@@ -204,11 +189,6 @@ public class DerivedFilterTranslator {
throw new IllegalArgumentException("unexpected column " + col);
}
- @Override
- public Object getValue(String field) {
- throw new UnsupportedOperationException();
- }
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
index 6d238ba..aedce4d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -22,17 +22,21 @@ import java.util.Collection;
import java.util.Set;
import org.apache.hadoop.hbase.util.Bytes;
-
-import com.google.common.collect.Sets;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.ISegment;
-import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.IEvaluatableTuple;
+import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.filter.TupleFilterSerializer.Decorator;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
+
+import com.google.common.collect.Sets;
/**
* @author yangli9
@@ -216,7 +220,7 @@ public class CoprocessorFilter {
return unstrictlyFilteredColumns;
}
- public boolean evaluate(ITuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple) {
if (filter == null)
return true;
else
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd0fbd5c/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java
index 72ae5de..288afa9 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverTuple.java
@@ -18,14 +18,11 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import java.util.List;
-
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.filter.IEvaluatableTuple;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
/**
* A special kind of tuple that exposes column value (dictionary ID) directly on
@@ -33,7 +30,7 @@ import org.apache.kylin.metadata.tuple.ITuple;
*
* @author yangli9
*/
-public class ObserverTuple implements ITuple {
+public class ObserverTuple implements IEvaluatableTuple {
final CoprocessorRowType type;
@@ -53,20 +50,6 @@ public class ObserverTuple implements ITuple {
}
}
- @Override
- public List<TblColRef> getAllColumns() {
- return type.columnsAsList;
- }
-
- @Override
- public Object[] getAllValues() {
- int n = type.getColumnCount();
- for (int i = 0; i < n; i++) {
- getValueAt(i);
- }
- return values;
- }
-
private String getValueAt(int i) {
int n = type.getColumnCount();
if (i < 0 || i >= n)
@@ -85,14 +68,4 @@ public class ObserverTuple implements ITuple {
return getValueAt(i);
}
- @Override
- public List<String> getAllFields() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object getValue(String field) {
- throw new UnsupportedOperationException();
- }
-
}
[29/50] incubator-kylin git commit: Merge branch 'staging' into
streaming
Posted by li...@apache.org.
Merge branch 'staging' into streaming
Conflicts:
metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bb83ca1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bb83ca1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bb83ca1c
Branch: refs/heads/streaming
Commit: bb83ca1c6edb4fc52fabda8a0506de0881a2d39d
Parents: 78e9ed0 2157fe5
Author: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Authored: Mon Mar 9 11:25:57 2015 +0100
Committer: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Committed: Mon Mar 9 11:25:57 2015 +0100
----------------------------------------------------------------------
bin/kylin.sh | 79 ++++
bin/start-kylin.sh | 55 ---
bin/stop-kylin.sh | 21 -
.../org/apache/kylin/cube/model/CubeDesc.java | 4 -
docs/Installation/install_and_run.md | 7 +-
docs/website/download/index.md | 19 +-
.../org/apache/kylin/jdbc/util/SQLTypeMap.java | 6 +-
.../kylin/metadata/model/DataModelDesc.java | 10 +-
.../kylin/metadata/model/PartitionDesc.java | 15 +-
.../metadata/tool/HiveSourceTableLoader.java | 13 +-
webapp/app/js/controllers/admin.js | 431 +++++++++----------
webapp/app/js/controllers/cubeEdit.js | 107 ++++-
webapp/app/js/controllers/cubeModel.js | 35 +-
webapp/app/js/controllers/cubeSchema.js | 15 -
webapp/app/js/controllers/cubes.js | 55 +--
webapp/app/js/controllers/page.js | 2 +-
webapp/app/js/model/jobListModel.js | 11 -
webapp/app/js/model/metaModel.js | 2 +-
.../cubeDesigner/advanced_settings.html | 3 +-
.../app/partials/cubeDesigner/dimensions.html | 1 -
.../app/partials/cubeDesigner/incremental.html | 12 +-
webapp/app/partials/jobs/job_merge.html | 2 +-
webapp/app/partials/jobs/job_refresh.html | 2 +-
webapp/app/partials/jobs/job_submit.html | 170 ++++----
webapp/app/partials/query/query.html | 4 +-
webapp/app/partials/query/query_detail.html | 17 +-
26 files changed, 598 insertions(+), 500 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb83ca1c/docs/Installation/install_and_run.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb83ca1c/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --cc metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 5a72386,c9271fc..cd0caee
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@@ -57,7 -54,7 +57,7 @@@ public class DataModelDesc extends Root
private String filterCondition;
@JsonProperty("partition_desc")
PartitionDesc partitionDesc;
--
++
@JsonProperty("capacity")
private RealizationCapacity capacity = RealizationCapacity.MEDIUM;
@@@ -96,7 -93,7 +96,6 @@@
public boolean isFactTable(String factTable) {
return this.factTable.equalsIgnoreCase(factTable);
}
--
public String getFilterCondition() {
return filterCondition;
@@@ -146,10 -143,13 +145,15 @@@
public void init(Map<String, TableDesc> tables) {
initJoinColumns(tables);
-
+ DimensionDesc.capicalizeStrings(dimensions);
+ initPartitionDesc(tables);
+ }
+
+ private void initPartitionDesc(Map<String, TableDesc> tables) {
- if(this.partitionDesc != null)
++ if (this.partitionDesc != null)
+ this.partitionDesc.init(tables);
}
+
private void initJoinColumns(Map<String, TableDesc> tables) {
// join columns may or may not present in cube;
// here we don't modify 'allColumns' and 'dimensionColumns';
[10/50] incubator-kylin git commit: KYLIN-624 Add dimension and
metric info into DataModel
Posted by li...@apache.org.
KYLIN-624 Add dimension and metric info into DataModel
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c3c47db3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c3c47db3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c3c47db3
Branch: refs/heads/streaming
Commit: c3c47db365f8e2dfa9cb29d2537b09942c70441e
Parents: def5e7e
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Mar 6 16:47:50 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Mar 6 16:47:50 2015 +0800
----------------------------------------------------------------------
.../test_kylin_inner_join_model_desc.json | 59 ++++++++++++++++++++
.../test_kylin_left_join_model_desc.json | 59 ++++++++++++++++++++
.../kylin/metadata/model/DataModelDesc.java | 20 +++++--
.../kylin/metadata/MetadataManagerTest.java | 19 +++++--
.../kylin/storage/hybrid/HybridInstance.java | 4 ++
5 files changed, 152 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index 655045a..86f8169 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -30,6 +30,65 @@
"foreign_key" : [ "SLR_SEGMENT_CD" ]
}
} ],
+ "dimensions": [
+ {
+ "table": "default.test_kylin_fact",
+ "columns": [
+ "lstg_format_name",
+ "LSTG_SITE_ID",
+ "SLR_SEGMENT_CD"
+ ]
+ },
+ {
+ "table": "default.test_category_groupings",
+ "columns": [
+ "leaf_categ_id",
+ "site_id",
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER",
+ "meta_categ_name",
+ "categ_lvl2_name",
+ "categ_lvl3_name"
+ ]
+ },
+ {
+ "table": "edw.test_sites",
+ "columns": [
+ "site_id",
+ "site_name",
+ "cre_user"
+ ]
+ },
+ {
+ "table": "edw.test_seller_type_dim",
+ "columns": [
+ "seller_type_cd",
+ "seller_type_desc"
+ ]
+ },
+ {
+ "table": "edw.test_cal_dt",
+ "columns": [
+ "cal_dt",
+ "week_beg_dt"
+ ]
+ },
+ {
+ "table": "default.test_kylin_fact",
+ "columns": [
+ "TRANS_ID",
+ "CAL_DT",
+ "LEAF_CATEG_ID",
+ "SELLER_ID"
+ ]
+ }
+ ],
+ "metrics": [
+ "PRICE",
+ "ITEM_COUNT"
+ ],
"capacity" : "MEDIUM",
"last_modified" : 1422435345352,
"fact_table" : "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index 1868c54..fa11c33 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -30,6 +30,65 @@
"foreign_key" : [ "SLR_SEGMENT_CD" ]
}
} ],
+ "dimensions": [
+ {
+ "table": "default.test_kylin_fact",
+ "columns": [
+ "lstg_format_name",
+ "LSTG_SITE_ID",
+ "SLR_SEGMENT_CD"
+ ]
+ },
+ {
+ "table": "default.test_category_groupings",
+ "columns": [
+ "leaf_categ_id",
+ "site_id",
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER",
+ "meta_categ_name",
+ "categ_lvl2_name",
+ "categ_lvl3_name"
+ ]
+ },
+ {
+ "table": "edw.test_sites",
+ "columns": [
+ "site_id",
+ "site_name",
+ "cre_user"
+ ]
+ },
+ {
+ "table": "edw.test_seller_type_dim",
+ "columns": [
+ "seller_type_cd",
+ "seller_type_desc"
+ ]
+ },
+ {
+ "table": "edw.test_cal_dt",
+ "columns": [
+ "cal_dt",
+ "week_beg_dt"
+ ]
+ },
+ {
+ "table": "default.test_kylin_fact",
+ "columns": [
+ "TRANS_ID",
+ "CAL_DT",
+ "LEAF_CATEG_ID",
+ "SELLER_ID"
+ ]
+ }
+ ],
+ "metrics": [
+ "PRICE",
+ "ITEM_COUNT"
+ ],
"capacity" : "MEDIUM",
"last_modified" : 1422435345352,
"fact_table" : "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 6d1bc65..5a72386 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -18,10 +18,7 @@
package org.apache.kylin.metadata.model;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
+import java.util.*;
import org.apache.commons.lang.ArrayUtils;
@@ -50,6 +47,12 @@ public class DataModelDesc extends RootPersistentEntity {
@JsonProperty("lookups")
private LookupDesc[] lookups;
+ @JsonProperty("dimensions")
+ private List<DimensionDesc> dimensions;
+
+ @JsonProperty("metrics")
+ private String[] metrics;
+
@JsonProperty("filter_condition")
private String filterCondition;
@JsonProperty("partition_desc")
@@ -143,6 +146,8 @@ public class DataModelDesc extends RootPersistentEntity {
public void init(Map<String, TableDesc> tables) {
initJoinColumns(tables);
+
+ DimensionDesc.capicalizeStrings(dimensions);
}
private void initJoinColumns(Map<String, TableDesc> tables) {
@@ -218,4 +223,11 @@ public class DataModelDesc extends RootPersistentEntity {
return ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
}
+ public List<DimensionDesc> getDimensions() {
+ return dimensions;
+ }
+
+ public String[] getMetrics() {
+ return metrics;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index e71294a..3b65d97 100644
--- a/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata;
import java.util.List;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.junit.After;
import org.junit.Assert;
@@ -28,6 +29,8 @@ import org.junit.Test;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import static org.apache.kylin.metadata.MetadataManager.*;
+
/**
* Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 2:38 PM To
* change this template use File | Settings | File Templates.
@@ -46,14 +49,14 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
@Test
public void testListAllTables() throws Exception {
- List<TableDesc> tables = MetadataManager.getInstance(getTestConfig()).listAllTables();
+ List<TableDesc> tables = getInstance(getTestConfig()).listAllTables();
Assert.assertNotNull(tables);
Assert.assertTrue(tables.size() > 0);
}
@Test
public void testFindTableByName() throws Exception {
- TableDesc table = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_CAL_DT");
+ TableDesc table = getInstance(getTestConfig()).getTableDesc("EDW.TEST_CAL_DT");
Assert.assertNotNull(table);
Assert.assertEquals("EDW.TEST_CAL_DT", table.getIdentity());
}
@@ -61,9 +64,15 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
@Test
public void testGetInstance() throws Exception {
- Assert.assertNotNull(MetadataManager.getInstance(getTestConfig()));
- Assert.assertNotNull(MetadataManager.getInstance(getTestConfig()).listAllTables());
- Assert.assertTrue(MetadataManager.getInstance(getTestConfig()).listAllTables().size() > 0);
+ Assert.assertNotNull(getInstance(getTestConfig()));
+ Assert.assertNotNull(getInstance(getTestConfig()).listAllTables());
+ Assert.assertTrue(getInstance(getTestConfig()).listAllTables().size() > 0);
}
+ @Test
+ public void testDataModel() throws Exception {
+ DataModelDesc modelDesc = getInstance(getTestConfig()).getDataModelDesc("test_kylin_left_join_model_desc");
+
+ Assert.assertTrue(modelDesc.getDimensions().size()>0);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c3c47db3/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index ddccf90..fcdee34 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -100,6 +100,10 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
this.name = name;
}
+ @Override
+ public String toString() {
+ return getCanonicalName();
+ }
@Override
public String getCanonicalName() {
[14/50] incubator-kylin git commit: Merge pull request #440 from
janzhongi/staging
Posted by li...@apache.org.
Merge pull request #440 from janzhongi/staging
enhance aggregation group generate logic,add mandatory removed logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cfbd6ae2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cfbd6ae2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cfbd6ae2
Branch: refs/heads/streaming
Commit: cfbd6ae29bdac822eec632e6c0eb0930f403ed28
Parents: e6b7e27 38aac2c
Author: Zhong,Jian <ji...@ebay.com>
Authored: Fri Mar 6 18:21:17 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Fri Mar 6 18:21:17 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 88 +++++++++++++-------
webapp/app/js/controllers/cubeSchema.js | 7 --
webapp/app/js/model/jobListModel.js | 11 ---
.../cubeDesigner/advanced_settings.html | 1 +
4 files changed, 60 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
[48/50] incubator-kylin git commit: Merge branch 'staging' into
streaming
Posted by li...@apache.org.
Merge branch 'staging' into streaming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0b71528f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0b71528f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0b71528f
Branch: refs/heads/streaming
Commit: 0b71528f6fe569fc12d1a609ac52240f90236b63
Parents: b25e227 54dc31f
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Mar 11 14:02:33 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Mar 11 14:02:33 2015 +0800
----------------------------------------------------------------------
docs/website/download/index.md | 2 +-
.../kylin/job/execution/AbstractExecutable.java | 1 +
.../kylin/job/hadoop/AbstractHadoopJob.java | 65 ++++++++++++++------
.../apache/kylin/metadata/MetadataManager.java | 1 +
4 files changed, 48 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
[23/50] incubator-kylin git commit: Merge branch 'staging' of
https://github.com/KylinOLAP/Kylin into staging
Posted by li...@apache.org.
Merge branch 'staging' of https://github.com/KylinOLAP/Kylin into staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f0207634
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f0207634
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f0207634
Branch: refs/heads/streaming
Commit: f020763477b14c86aee3f3c6b8c2355638a7e55b
Parents: afff4e7 a9bf9a3
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Mon Mar 9 17:10:31 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Mon Mar 9 17:10:31 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 2 +-
webapp/app/js/controllers/cubeModel.js | 35 ++++++++++++++++----
webapp/app/js/model/metaModel.js | 2 +-
.../app/partials/cubeDesigner/dimensions.html | 1 -
.../app/partials/cubeDesigner/incremental.html | 10 +++---
5 files changed, 36 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[19/50] incubator-kylin git commit: Merge pull request #442 from
janzhongi/staging
Posted by li...@apache.org.
Merge pull request #442 from janzhongi/staging
fix KYLIN-524 & add tip for partition column field
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4be14bb6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4be14bb6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4be14bb6
Branch: refs/heads/streaming
Commit: 4be14bb659abf341b7e1208933a0a21e80707033
Parents: cfbd6ae c6adb69
Author: Zhong,Jian <ji...@ebay.com>
Authored: Mon Mar 9 16:28:11 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Mon Mar 9 16:28:11 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeModel.js | 35 ++++++++++++++++----
webapp/app/js/model/metaModel.js | 2 +-
.../app/partials/cubeDesigner/dimensions.html | 1 -
.../app/partials/cubeDesigner/incremental.html | 9 ++---
4 files changed, 34 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
[06/50] incubator-kylin git commit: update download page
Posted by li...@apache.org.
update download page
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fcae14e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fcae14e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fcae14e9
Branch: refs/heads/streaming
Commit: fcae14e9a8c9cc6fb0f25ebaedb06af39800d167
Parents: c9147ca
Author: lukehan <lu...@apache.org>
Authored: Fri Mar 6 14:55:01 2015 +0800
Committer: lukehan <lu...@apache.org>
Committed: Fri Mar 6 14:55:01 2015 +0800
----------------------------------------------------------------------
docs/website/download/index.md | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fcae14e9/docs/website/download/index.md
----------------------------------------------------------------------
diff --git a/docs/website/download/index.md b/docs/website/download/index.md
index 1c6b935..da40c6e 100644
--- a/docs/website/download/index.md
+++ b/docs/website/download/index.md
@@ -13,21 +13,28 @@ title: Community
<div class="row" style="margin-top:-20px;">
<div class="col-sm-4 col-md-4">
- <p class="big_text">Binary Package</p>
+ <p class="big_text">Binary Package (v0.7.x)</p>
<p><a href="http://kylin.incubator.apache.org/download/kylin-0.7.1-SNAPSHOT.tgz" target="_blank">v0.7.1-SNAPSHOT</a>
- <br/>date - Feb 12, 2015 <br/></p>
+ <br/>Updated Date - March 2, 2015 <br/>
+ </p>
</div>
<div class="col-sm-4 col-md-4">
<p class="big_text">Source Code Repo</p>
- <p>v0.6.6 (Master branch): <a href="https://github.com/apache/incubator-kylin" target="_blank"> https://github.com/apache/incubator-kylin</a></p>
- <p>v0.7.1 (Inverted-Index branch): <a href="https://github.com/apache/incubator-kylin/tree/inverted-index" target="_blank">https://github.com/apache/incubator-kylin/tree/inverted-index</a></p>
+ <p>v0.7.1 (master branch):
+ <a href="https://github.com/apache/incubator-kylin" target="_blank" >
+ https://github.com/apache/incubator-kylin
+ </a></p>
+ <p>v0.6.6 (v0.6 branch): <a href="https://github.com/apache/incubator-kylin/tree/0.6" target="_blank"> https://github.com/apache/incubator-kylin/tree/0.6</a></p>
+
</div>
<div class="col-sm-4 col-md-4">
- <p class="big_text">Installation Guide</p>
+ <p class="big_text">Installation Guide</p>
+ <p>v0.7.x: <a href="https://github.com/KylinOLAP/Kylin/blob/master/docs/Installation/install_and_run.md" target="_blank">https://github.com/KylinOLAP/Kylin/blob/master/docs/Installation/install_and_run.md</a></p>
+
<p>v0.6.x: <a href="https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-CLI-installation" target="_blank">https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-CLI-installation</a></p>
- <p>v0.7.x: <a href="https://github.com/KylinOLAP/Kylin/blob/inverted-index/docs/Installation/install_and_run.md" target="_blank">https://github.com/KylinOLAP/Kylin/blob/inverted-index/docs/Installation/install_and_run.md</a></p>
+
</div>
</div>
[09/50] incubator-kylin git commit: Rename IIDimension to
DimensionDesc, and move it to metadata module.
Posted by li...@apache.org.
Rename IIDimension to DimensionDesc, and move it to metadata module.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/def5e7e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/def5e7e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/def5e7e6
Branch: refs/heads/streaming
Commit: def5e7e6c43d9fa619ebe6c94892c97e41162b79
Parents: 13f3260
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Mar 6 15:46:47 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Mar 6 15:46:47 2015 +0800
----------------------------------------------------------------------
.../kylin/invertedindex/model/IIDesc.java | 19 +++---
.../kylin/invertedindex/model/IIDimension.java | 69 --------------------
.../kylin/metadata/model/DimensionDesc.java | 69 ++++++++++++++++++++
3 files changed, 78 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/def5e7e6/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
index aae751b..e094dfc 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
@@ -26,7 +26,6 @@ import java.util.HashSet;
import java.util.List;
import org.apache.commons.net.util.Base64;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.util.Bytes;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -67,9 +66,9 @@ public class IIDesc extends RootPersistentEntity {
@JsonProperty("timestamp_dimension")
private String timestampDimension;
@JsonProperty("bitmap_dimensions")
- private List<IIDimension> bitmapDimensions;
+ private List<DimensionDesc> bitmapDimensions;
@JsonProperty("value_dimensions")
- private List<IIDimension> valueDimensions;
+ private List<DimensionDesc> valueDimensions;
@JsonProperty("metrics")
private String[] metricNames;
@JsonProperty("sharding")
@@ -107,17 +106,17 @@ public class IIDesc extends RootPersistentEntity {
timestampDimension = timestampDimension.toUpperCase();
// capitalize
- IIDimension.capicalizeStrings(bitmapDimensions);
- IIDimension.capicalizeStrings(valueDimensions);
+ DimensionDesc.capicalizeStrings(bitmapDimensions);
+ DimensionDesc.capicalizeStrings(valueDimensions);
StringUtil.toUpperCaseArray(metricNames, metricNames);
// retrieve all columns and all tables, and make available measure to ii
HashSet<String> allTableNames = Sets.newHashSet();
measureDescs = Lists.newArrayList();
measureDescs.add(makeCountMeasure());
- for (IIDimension iiDimension : Iterables.concat(bitmapDimensions, valueDimensions)) {
- TableDesc tableDesc = this.getTableDesc(iiDimension.getTable());
- for (String column : iiDimension.getColumns()) {
+ for (DimensionDesc dimensionDesc : Iterables.concat(bitmapDimensions, valueDimensions)) {
+ TableDesc tableDesc = this.getTableDesc(dimensionDesc.getTable());
+ for (String column : dimensionDesc.getColumns()) {
ColumnDesc columnDesc = tableDesc.findColumnByName(column);
TblColRef tcr = new TblColRef(columnDesc);
allColumns.add(tcr);
@@ -144,8 +143,8 @@ public class IIDesc extends RootPersistentEntity {
}
// indexing for each type of columns
- bitmapCols = new int[IIDimension.getColumnCount(bitmapDimensions)];
- valueCols = new int[IIDimension.getColumnCount(valueDimensions)];
+ bitmapCols = new int[DimensionDesc.getColumnCount(bitmapDimensions)];
+ valueCols = new int[DimensionDesc.getColumnCount(valueDimensions)];
metricsCols = new int[metricNames.length];
metricsColSet = new BitSet(this.getTableDesc(this.getFactTableName()).getColumnCount());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/def5e7e6/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
deleted file mode 100644
index ee8b6b9..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.model;
-
-import java.util.List;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.common.util.StringUtil;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 12/26/14.
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class IIDimension {
- @JsonProperty("table")
- private String table;
- @JsonProperty("columns")
- private String[] columns;
-
- public String getTable() {
- return table;
- }
-
- public void setTable(String table) {
- this.table = table;
- }
-
- public String[] getColumns() {
- return columns;
- }
-
- public void setColumns(String[] columns) {
- this.columns = columns;
- }
-
-
- public static void capicalizeStrings(List<IIDimension> dimensions) {
- for (IIDimension iiDimension : dimensions) {
- iiDimension.setTable(iiDimension.getTable().toUpperCase());
- StringUtil.toUpperCaseArray(iiDimension.getColumns(), iiDimension.getColumns());
- }
- }
-
- public static int getColumnCount(List<IIDimension> iiDimensions) {
- int count = 0;
- for (IIDimension iiDimension : iiDimensions) {
- count += iiDimension.getColumns().length;
- }
- return count;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/def5e7e6/metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java
new file mode 100644
index 0000000..1cc6235
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.common.util.StringUtil;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 12/26/14.
+ */
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
+public class DimensionDesc {
+ @JsonProperty("table")
+ private String table;
+ @JsonProperty("columns")
+ private String[] columns;
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public String[] getColumns() {
+ return columns;
+ }
+
+ public void setColumns(String[] columns) {
+ this.columns = columns;
+ }
+
+
+ public static void capicalizeStrings(List<DimensionDesc> dimensions) {
+ for (DimensionDesc dimensionDesc : dimensions) {
+ dimensionDesc.setTable(dimensionDesc.getTable().toUpperCase());
+ StringUtil.toUpperCaseArray(dimensionDesc.getColumns(), dimensionDesc.getColumns());
+ }
+ }
+
+ public static int getColumnCount(List<DimensionDesc> dimensionDescs) {
+ int count = 0;
+ for (DimensionDesc dimensionDesc : dimensionDescs) {
+ count += dimensionDesc.getColumns().length;
+ }
+ return count;
+ }
+
+}
[02/50] incubator-kylin git commit: Merge branch 'streaming' of
https://github.com/KylinOLAP/Kylin into streaming
Posted by li...@apache.org.
Merge branch 'streaming' of https://github.com/KylinOLAP/Kylin into streaming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9609c56b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9609c56b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9609c56b
Branch: refs/heads/streaming
Commit: 9609c56b8005908d7ead0e09fb64ffdb67a001ae
Parents: d7cf8a7 22dbc12
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Mar 6 13:29:35 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Mar 6 13:29:35 2015 +0800
----------------------------------------------------------------------
pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
[13/50] incubator-kylin git commit: enhance aggregation group
generate logic, add mandatory removed logic
Posted by li...@apache.org.
enhance aggregation group generate logic,add mandatory removed logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/38aac2c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/38aac2c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/38aac2c0
Branch: refs/heads/streaming
Commit: 38aac2c09360efe5df14c2229eafec15a506133f
Parents: 21dc31a
Author: jiazhong <ji...@ebay.com>
Authored: Fri Mar 6 18:20:21 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Fri Mar 6 18:20:21 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 88 +++++++++++++-------
webapp/app/js/controllers/cubeSchema.js | 7 --
webapp/app/js/model/jobListModel.js | 11 ---
.../cubeDesigner/advanced_settings.html | 1 +
4 files changed, 60 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/38aac2c0/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index e903f65..d1fb103 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -255,6 +255,17 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
}
}
+ $scope.updateMandatory = function(rowkey_column){
+ if(!rowkey_column.mandatory){
+ angular.forEach($scope.cubeMetaFrame.rowkey.aggregation_groups, function (group, index) {
+ var index = group.indexOf(rowkey_column.column);
+ if(index>-1){
+ group.splice(index,1);
+ }
+ });
+ }
+ }
+
function reGenerateRowKey(){
$log.log("reGen rowkey & agg group");
var tmpRowKeyColumns = [];
@@ -328,6 +339,16 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
});
+
+ //rm mandatory column from aggregation item
+ angular.forEach($scope.cubeMetaFrame.rowkey.rowkey_columns,function(value,index){
+ if(value.mandatory){
+ tmpAggregationItems = _.filter(tmpAggregationItems,function(item){
+ return item!=value.column;
+ });
+ }
+ });
+
var rowkeyColumns = $scope.cubeMetaFrame.rowkey.rowkey_columns;
var newRowKeyColumns = sortSharedData(rowkeyColumns,tmpRowKeyColumns);
var increasedColumns = increasedColumn(rowkeyColumns,tmpRowKeyColumns);
@@ -368,7 +389,10 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
if($scope.cubeMode==="addNewCube"){
- if(!tmpAggregationItems.length) return;
+ if(!tmpAggregationItems.length) {
+ $scope.cubeMetaFrame.rowkey.aggregation_groups=[];
+ return;
+ }
var newUniqAggregationItem = [];
angular.forEach(tmpAggregationItems, function (item, index) {
@@ -377,52 +401,58 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
}
});
- var hierarchyItems = [];
- for(var m=0;m<hierarchyItemArray.length;m++){
- for(var n=0;n<hierarchyItemArray[m].length;n++){
- hierarchyItems.push(hierarchyItemArray[m][n]);
- }
- }
+ var hierarchyItems = hierarchyItemArray.join().split(",");
var unHierarchyItems = increasedData(hierarchyItems,newUniqAggregationItem);
-// hierarchyItems
+ //hierarchyItems
var increasedDataGroups = sliceGroupItemToGroups(unHierarchyItems);
+ if(!hierarchyItemArray.length){
+ $scope.cubeMetaFrame.rowkey.aggregation_groups = increasedDataGroups;
+ return;
+ };
+
var lastAggregationGroup = increasedDataGroups.length===0?[]:increasedDataGroups[increasedDataGroups.length-1];
+
if(lastAggregationGroup.length<10){
if(lastAggregationGroup.length+hierarchyItemArray.length<=10){
lastAggregationGroup = lastAggregationGroup.concat(hierarchyItems);
if(increasedDataGroups.length==0){
+ //case only hierarchy
increasedDataGroups[0]=lastAggregationGroup;
}else{
increasedDataGroups[increasedDataGroups.length-1]=lastAggregationGroup;
}
}
else{
- var spareLength = 10-lastAggregationGroup.length;
- var partialHierarchy =lastAggregationGroup.concat(hierarchyItemArray.slice(0,spareLength));
- //add hierarchy to last group and make sure length less than 10
- lastAggregationGroup = lastAggregationGroup.concat(partialHierarchy);
- var spareHierarchy = hierarchyItemArray.slice(spareLength+1,hierarchyItemArray.length-1);
- if(spareHierarchy.length){
- lastAggregationGroup.push(spareHierarchy);
- }
- increasedDataGroups[increasedDataGroups.length-1]=lastAggregationGroup;
+ var cutIndex = 10-lastAggregationGroup.length;
+ var partialHierarchy =hierarchyItemArray.slice(0,cutIndex).join().split(",");
+ //add hierarchy to last group and make sure length less than 10
+ lastAggregationGroup = lastAggregationGroup.concat(partialHierarchy);
+ increasedDataGroups[increasedDataGroups.length-1]=lastAggregationGroup;
+ var leftHierarchy = hierarchyItemArray.slice(cutIndex);
+
+ var leftHierarchyLength = leftHierarchy.length;
+ var grpLength = parseInt(leftHierarchyLength/10);
+ if(leftHierarchyLength%10==0&&leftHierarchyLength!=0){
+ grpLength--;
+ }
+ for(var i=0;i<=grpLength;i++){
+ var hierAggGroupUnit = leftHierarchy.slice(i*10,(i+1)*10).join().split(",");
+ increasedDataGroups.push(hierAggGroupUnit);
+ }
}
}
+ //lastAggregationGroup length >=10
else{
- if(hierarchyItemArray.length<=10){
- increasedDataGroups.push(hierarchyItems);
- }else{
- var partialHierarchy =lastAggregationGroup.concat(hierarchyItemArray.slice(0,10));
- //add hierarchy to last group and make sure length less than 10
- increasedDataGroups.push(partialHierarchy);
- var spareHierarchy = hierarchyItemArray.slice(11,hierarchyItemArray.length-1);
- if(spareHierarchy.length){
- increasedDataGroups.push(spareHierarchy);
+ var hierrachyArrayLength = hierarchyItemArray.length;
+ var grpLength = parseInt(hierrachyArrayLength/10);
+ if(hierrachyArrayLength%10==0&&hierrachyArrayLength!=0){
+ grpLength--;
+ }
+ for(var i=0;i<=grpLength;i++){
+ var hierAggGroupUnit = hierarchyItemArray.slice(i*10,(i+1)*10).join().split(",");
+ increasedDataGroups.push(hierAggGroupUnit);
}
- }
}
-
-
//! here get the latest aggregation groups,only effect when add newCube
$scope.cubeMetaFrame.rowkey.aggregation_groups = increasedDataGroups;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/38aac2c0/webapp/app/js/controllers/cubeSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js
index b0de094..d883b59 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -62,13 +62,6 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic
ModelService.get({model_name: $scope.cubeMetaFrame.model_name}, function (model) {
if (model) {
$scope.metaModel.model = model;
-
- //convert GMT mills ,to make sure partition date show GMT Date
- //should run only one time
- //if($scope.metaModel.model.partition_desc&&$scope.metaModel.model.partition_desc.partition_date_start)
- //{
- // $scope.metaModel.model.partition_desc.partition_date_start+=new Date().getTimezoneOffset()*60000;
- //}
}
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/38aac2c0/webapp/app/js/model/jobListModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/jobListModel.js b/webapp/app/js/model/jobListModel.js
index 166e4e4..764791a 100755
--- a/webapp/app/js/model/jobListModel.js
+++ b/webapp/app/js/model/jobListModel.js
@@ -36,10 +36,6 @@ KylinApp.service('JobList',function(JobService,$q){
}
});
-// $scope.state.loading = false;
-// if (angular.isDefined($scope.state.selectedJob)) {
-// $scope.state.selectedJob = $scope.jobs[selectedJob.uuid];
-// }
defer.resolve(jobs.length);
});
@@ -47,13 +43,6 @@ KylinApp.service('JobList',function(JobService,$q){
};
-// this.removeCube = function(cube){
-// var cubeIndex = _this.cubes.indexOf(cube);
-// if (cubeIndex > -1) {
-// _this.cubes.splice(cubeIndex, 1);
-// }
-// }
-
this.removeAll = function(){
_this.jobs=[];
};
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/38aac2c0/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index e29e6a4..dfa05c6 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -119,6 +119,7 @@
<button type="button " ng-if="state.mode=='edit'"
class="btn btn-xs btn-default {{rowkey_column.mandatory? 'active':''}}"
ng-model="rowkey_column.mandatory"
+ ng-click="updateMandatory(rowkey_column);"
btn-checkbox btn-checkbox-true="true" btn-checkbox-false="false">
{{rowkey_column.mandatory? 'Y':'N'}}
</button>
[08/50] incubator-kylin git commit: Update install_and_run.md
Posted by li...@apache.org.
Update install_and_run.md
update start/stop shell
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e6b7e279
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e6b7e279
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e6b7e279
Branch: refs/heads/streaming
Commit: e6b7e2796ab5123c1324cedbfb2f035dd7c0b634
Parents: ef27f3a
Author: Luke Han <lu...@users.noreply.github.com>
Authored: Fri Mar 6 15:03:17 2015 +0800
Committer: Luke Han <lu...@users.noreply.github.com>
Committed: Fri Mar 6 15:03:17 2015 +0800
----------------------------------------------------------------------
docs/Installation/install_and_run.md | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6b7e279/docs/Installation/install_and_run.md
----------------------------------------------------------------------
diff --git a/docs/Installation/install_and_run.md b/docs/Installation/install_and_run.md
index d634b32..82369eb 100644
--- a/docs/Installation/install_and_run.md
+++ b/docs/Installation/install_and_run.md
@@ -5,8 +5,8 @@
1. Download the release version(according to the hadoop distribution)
2. Setup a KYLIN_HOME pointing to the corresponding directory where you extract the release tar
3. Make sure the user has the privilege to run hadoop, hive and hbase cmd in shell. If you are not so sure, you can just run **bin/check-env.sh**, it will print out the detail information if you have some environment issues.
-4. To start Kylin, simply run **bin/start-kylin.sh**
-5. To stop Kylin, simply run **bin/stop-kylin.sh**
+4. To start Kylin, simply run **bin/kylin.sh start**
+5. To stop Kylin, simply run **bin/kylin.sh stop**
If you are running Kylin in a cluster or you have multiple Kylin instances, please make sure you have the following property correctly configured.
@@ -27,8 +27,7 @@ If you are running Kylin in a cluster or you have multiple Kylin instances, plea
> │ ├── check-env.sh
> │ ├── find-hive-dependency.sh
> │ ├── health-check.sh
-> │ ├── start-kylin.sh
-> │ └── stop-kylin.sh
+> │ └── kylin.sh
> │
> ├── conf
> │ ├── kylin_job_conf.xml
[50/50] incubator-kylin git commit: KYLIN-638 kylin.sh stop not
working
Posted by li...@apache.org.
KYLIN-638 kylin.sh stop not working
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/624c4347
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/624c4347
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/624c4347
Branch: refs/heads/streaming
Commit: 624c4347feee9e3bcfca5ae7e8538abe82fbf152
Parents: 0b13251
Author: honma <ho...@ebay.com>
Authored: Thu Mar 12 16:36:40 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Thu Mar 12 16:36:40 2015 -0700
----------------------------------------------------------------------
bin/kylin.sh | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/624c4347/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
index 7981bb8..7941093 100644
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -1,9 +1,11 @@
#!/bin/sh
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+
if [ $1 == "start" ]
then
- dir=$(dirname ${0})
- source ${dir}/check-env.sh
+
tomcat_root=${dir}/../tomcat
export tomcat_root
[03/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4a59d38f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4a59d38f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4a59d38f
Branch: refs/heads/streaming
Commit: 4a59d38ff67407e38521f6ab7061b10ca2c57df4
Parents: 3d3e312
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 6 14:09:08 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 6 14:09:08 2015 +0800
----------------------------------------------------------------------
bin/kylin.sh | 76 +++++++++++++++++++++++++++++++++++++++++++++++++
bin/start-kylin.sh | 55 -----------------------------------
bin/stop-kylin.sh | 21 --------------
3 files changed, 76 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4a59d38f/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
new file mode 100644
index 0000000..4efaf9e
--- /dev/null
+++ b/bin/kylin.sh
@@ -0,0 +1,76 @@
+#!/bin/sh
+
+if [ $1 == "start" ]
+then
+ dir=$(dirname ${0})
+ source ${dir}/check-env.sh
+ tomcat_root=${dir}/../tomcat
+ export tomcat_root
+
+ #if [ ! -z "$KYLIN_LD_LIBRARY_PATH" ]
+ #then
+ # echo "KYLIN_LD_LIBRARY_PATH is set to $KYLIN_LD_LIBRARY_PATH"
+ #else
+ # exit 1
+ #fi
+
+ #The location of all hadoop/hbase configurations are difficult to get.
+ #Plus, some of the system properties are secretly set in hadoop/hbase shell command.
+ #For example, in hdp 2.2, there is a system property called hdp.version,
+ #which we cannot get until running hbase or hadoop shell command.
+ #
+ #To save all these troubles, we use hbase runjar to start tomcat.
+ #In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
+ #hbase command will do all the dirty tasks for us:
+
+ #-Djava.library.path=${KYLIN_LD_LIBRARY_PATH} \
+
+
+ useSandbox=`cat ${KYLIN_HOME}/conf/kylin.properties | grep 'kylin.sandbox' | awk -F '=' '{print $2}'`
+ spring_profile="default"
+ if [ "$useSandbox" = "true" ]
+ then spring_profile="sandbox"
+ fi
+ source ${dir}/find-hive-dependency.sh
+
+ export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
+ export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
+
+ hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
+ -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
+ -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
+ -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
+ -Djava.endorsed.dirs=${tomcat_root}/endorsed \
+ -Dcatalina.base=${tomcat_root} \
+ -Dcatalina.home=${tomcat_root} \
+ -Djava.io.tmpdir=${tomcat_root}/temp \
+ -Dkylin.hive.dependency=${hive_dependency} \
+ -Dspring.profiles.active=${spring_profile} \
+ org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin.log 2>&1 & echo $! > ${KYLIN_HOME}/pid &
+ echo "A new Kylin instance is started by $USER, stop it using \"stop-kylin.sh\""
+ if [ "$useSandbox" = "true" ]
+ then echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
+ else
+ echo "Please visit http://<ip>:7070/kylin"
+ fi
+ echo "You can check the log at ${tomcat_root}/logs/kylin.log"
+elif [ $1 == "stop" ]
+then
+ if [ ! -f "${KYLIN_HOME}/pid" ]
+ then
+ echo "kylin is not running, please check"
+ exit 1
+ fi
+ pid=`cat ${KYLIN_HOME}/pid`
+ if [ "$pid" = "" ]
+ then
+ echo "kylin is not running, please check"
+ exit 1
+ else
+ echo "stopping kylin:$pid"
+ kill $pid
+ fi
+ rm ${KYLIN_HOME}/pid
+else
+ echo "usage: kylin.sh start or kylin.sh stop"
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4a59d38f/bin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/start-kylin.sh b/bin/start-kylin.sh
deleted file mode 100644
index 70cbb62..0000000
--- a/bin/start-kylin.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/sh
-
-dir=$(dirname ${0})
-source ${dir}/check-env.sh
-
-tomcat_root=${dir}/../tomcat
-export tomcat_root
-
-#if [ ! -z "$KYLIN_LD_LIBRARY_PATH" ]
-#then
-# echo "KYLIN_LD_LIBRARY_PATH is set to $KYLIN_LD_LIBRARY_PATH"
-#else
-# exit 1
-#fi
-
-#The location of all hadoop/hbase configurations are difficult to get.
-#Plus, some of the system properties are secretly set in hadoop/hbase shell command.
-#For example, in hdp 2.2, there is a system property called hdp.version,
-#which we cannot get until running hbase or hadoop shell command.
-#
-#To save all these troubles, we use hbase runjar to start tomcat.
-#In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
-#hbase command will do all the dirty tasks for us:
-
-#-Djava.library.path=${KYLIN_LD_LIBRARY_PATH} \
-
-
-useSandbox=`cat ${KYLIN_HOME}/conf/kylin.properties | grep 'kylin.sandbox' | awk -F '=' '{print $2}'`
-spring_profile="default"
-if [ "$useSandbox" = "true" ]
- then spring_profile="sandbox"
-fi
-source ${dir}/find-hive-dependency.sh
-
-export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
-export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
-
-hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
--Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
--Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
--Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
--Djava.endorsed.dirs=${tomcat_root}/endorsed \
--Dcatalina.base=${tomcat_root} \
--Dcatalina.home=${tomcat_root} \
--Djava.io.tmpdir=${tomcat_root}/temp \
--Dkylin.hive.dependency=${hive_dependency} \
--Dspring.profiles.active=${spring_profile} \
-org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin.log 2>&1 & echo $! > ${KYLIN_HOME}/pid &
-echo "A new Kylin instance is started by $USER, stop it using \"stop-kylin.sh\""
-if [ "$useSandbox" = "true" ]
- then echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
-else
- echo "Please visit http://<ip>:7070/kylin"
-fi
-echo "You can check the log at ${tomcat_root}/logs/kylin.log"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4a59d38f/bin/stop-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/stop-kylin.sh b/bin/stop-kylin.sh
deleted file mode 100644
index dad0a83..0000000
--- a/bin/stop-kylin.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/sh
-
-dir=$(dirname ${0})
-source ${dir}/check-env.sh
-
-if [ ! -f "${KYLIN_HOME}/pid" ]
-then
- echo "no pid found"
- exit 1
-fi
-
-pid=`cat ${KYLIN_HOME}/pid`
-if [ "$pid" = "" ]
-then
- echo "pid is empty"
- exit 1
-else
- echo "stopping kylin:$pid"
- kill $pid
-fi
-rm ${KYLIN_HOME}/pid
[41/50] incubator-kylin git commit: Merge branch 'staging' into
streaming
Posted by li...@apache.org.
Merge branch 'staging' into streaming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e3c50724
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e3c50724
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e3c50724
Branch: refs/heads/streaming
Commit: e3c50724379202e725a7d7c9d25f293009400ebf
Parents: bb83ca1 5f9e2c9
Author: jiazhong <ji...@ebay.com>
Authored: Tue Mar 10 18:20:16 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Tue Mar 10 18:20:16 2015 +0800
----------------------------------------------------------------------
.../kylin/job/execution/AbstractExecutable.java | 10 +-
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 -
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/js/listeners.js | 7 -
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
14 files changed, 115 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
[37/50] incubator-kylin git commit: fix filter 0 bytes bug ,
when bytes is 0 show '0 bytes' KYLIN-572
Posted by li...@apache.org.
fix filter 0 bytes bug
, when bytes is 0 show '0 bytes' KYLIN-572
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7dae30ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7dae30ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7dae30ef
Branch: refs/heads/streaming
Commit: 7dae30ef66b4132b84b4fcc4b635b838af5aaffb
Parents: ec0fb0f
Author: jiazhong <ji...@ebay.com>
Authored: Tue Mar 10 15:25:37 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Tue Mar 10 15:25:37 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/filters/filter.js | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7dae30ef/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 4c30905..f1bd991 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -27,6 +27,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
}
}
);
+ //edit model
if($scope.state.mode==="edit") {
$scope.metaModel = MetaModel;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7dae30ef/webapp/app/js/filters/filter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index f3364c7..555e31d 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -83,6 +83,7 @@ KylinApp
// Convert bytes into human readable format.
.filter('bytes', function() {
return function(bytes, precision) {
+ if (bytes === 0) { return '0 bytes' };
if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) {
return '-';
}
[20/50] incubator-kylin git commit: support 'string' and 'date' type
column as options for partition column
Posted by li...@apache.org.
support 'string' and 'date' type column as options for partition column
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f983e6cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f983e6cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f983e6cc
Branch: refs/heads/streaming
Commit: f983e6cc0ced65349833eac7053b34411c651a88
Parents: c6adb69
Author: jiazhong <ji...@ebay.com>
Authored: Mon Mar 9 17:09:14 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Mar 9 17:09:14 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 2 +-
webapp/app/partials/cubeDesigner/incremental.html | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f983e6cc/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index d1fb103..356a5b5 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -38,7 +38,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
$scope.getPartitonColumns = function(tableName){
var columns = _.filter($scope.getColumnsByTable(tableName),function(column){
- return column.datatype==="date";
+ return column.datatype==="date"||column.datatype==="string";
});
return columns;
};
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f983e6cc/webapp/app/partials/cubeDesigner/incremental.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/incremental.html b/webapp/app/partials/cubeDesigner/incremental.html
index 6b21dc3..2d48ee5 100755
--- a/webapp/app/partials/cubeDesigner/incremental.html
+++ b/webapp/app/partials/cubeDesigner/incremental.html
@@ -50,7 +50,7 @@
ng-options="metaModel.model.fact_table+'.'+columns.name as metaModel.model.fact_table+'.'+columns.name for columns in getPartitonColumns(metaModel.model.fact_table)" >
<option value="">--Select Partition Column--</option>
</select>
- <p class="text-red" ng-if="state.mode=='edit'">(Partition column will select 'date' type column from fact table)</p>
+ <p class="text-red" ng-if="state.mode=='edit'">(data format in column should be 'YYYY-MM-DD')</p>
<span ng-if="state.mode=='view'">
{{!!(metaModel.model.partition_desc.partition_date_column)?metaModel.model.partition_desc.partition_date_column: ''}}</span>
</div>
@@ -83,6 +83,7 @@
<div class="col-xs-12">
<ol class="text-info">
<li>Not required,leave as default if this cube always need full build</li>
+ <li>Partition column will select 'date' or 'string' type column from fact table</li>
<li>If column selected,please indicate start date to just pull certain data from source</li>
</ol>
</div>
[36/50] incubator-kylin git commit: Add more logs in hadoop job
Posted by li...@apache.org.
Add more logs in hadoop job
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/93f1f26e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/93f1f26e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/93f1f26e
Branch: refs/heads/streaming
Commit: 93f1f26e3e1f07ecd3919517a82c0bb9faadc4a3
Parents: 6074fb4
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Tue Mar 10 15:21:12 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Tue Mar 10 15:21:12 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/execution/AbstractExecutable.java | 1 +
.../main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/93f1f26e/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index c0c6160..4e15709 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -98,6 +98,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
onExecuteStart(executableContext);
result = doWork(executableContext);
} catch (Throwable e) {
+ logger.error("error running Executable", e);
onExecuteError(e, executableContext);
throw new ExecuteException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/93f1f26e/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index a8dc687..ff19357 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -161,6 +161,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
}
}
+ logger.info("Hadoop job classpath is: " + job.getConfiguration().get(MAP_REDUCE_CLASSPATH));
}
public void addInputDirs(String input, Job job) throws IOException {
[18/50] incubator-kylin git commit: fix KYLIN-524 & add tip for
partition column field
Posted by li...@apache.org.
fix KYLIN-524 & add tip for partition column field
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c6adb698
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c6adb698
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c6adb698
Branch: refs/heads/streaming
Commit: c6adb698d6b38d19fc156bb8f0ecff1cf90b88ec
Parents: cfbd6ae
Author: jiazhong <ji...@ebay.com>
Authored: Mon Mar 9 16:26:58 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Mar 9 16:26:58 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeModel.js | 35 ++++++++++++++++----
webapp/app/js/model/metaModel.js | 2 +-
.../app/partials/cubeDesigner/dimensions.html | 1 -
.../app/partials/cubeDesigner/incremental.html | 9 ++---
4 files changed, 34 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c6adb698/webapp/app/js/controllers/cubeModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeModel.js b/webapp/app/js/controllers/cubeModel.js
index 45425c8..1e5fab0 100644
--- a/webapp/app/js/controllers/cubeModel.js
+++ b/webapp/app/js/controllers/cubeModel.js
@@ -18,11 +18,8 @@
'use strict';
-KylinApp.controller('CubeModelCtrl', function ($scope, $modal,cubeConfig,ModelService,MetaModel) {
+KylinApp.controller('CubeModelCtrl', function ($scope, $modal,cubeConfig,ModelService,MetaModel,SweetAlert) {
-// if($scope.state.mode==="edit") {
-// $scope.metaModel = MetaModel;
-// }
$scope.cubeConfig = cubeConfig;
var DataModel = function () {
return {
@@ -113,9 +110,33 @@ KylinApp.controller('CubeModelCtrl', function ($scope, $modal,cubeConfig,ModelSe
$scope.resetParams();
};
- $scope.removeLookup = function (lookup) {
- lookupList.splice(lookupList.indexOf(lookup), 1);
- };
+ $scope.removeLookup = function (lookup) {
+ var dimExist = _.some($scope.cubeMetaFrame.dimensions,function(item,index){
+ return item.table===lookup.table;
+ });
+ if(dimExist) {
+ SweetAlert.swal({
+ title: '',
+ text: "Once it's removed, all relative dimensions will be removed. Are you sure to remove the lookup table?",
+ type: '',
+ showCancelButton: true,
+ confirmButtonColor: '#DD6B55',
+ confirmButtonText: "Yes",
+ closeOnConfirm: true
+ }, function (isConfirm) {
+ if (isConfirm) {
+ for (var i = $scope.cubeMetaFrame.dimensions.length - 1; i >= 0; i--) {
+ if ($scope.cubeMetaFrame.dimensions[i].table === lookup.table) {
+ $scope.cubeMetaFrame.dimensions.splice(i, 1);
+ }
+ }
+ lookupList.splice(lookupList.indexOf(lookup), 1);
+ }
+ });
+ }else{
+ lookupList.splice(lookupList.indexOf(lookup), 1);
+ }
+ };
$scope.resetParams = function () {
$scope.lookupState.editing = false;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c6adb698/webapp/app/js/model/metaModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/metaModel.js b/webapp/app/js/model/metaModel.js
index 9fd5ed6..a21c83c 100644
--- a/webapp/app/js/model/metaModel.js
+++ b/webapp/app/js/model/metaModel.js
@@ -70,7 +70,7 @@ KylinApp.service('MetaModel',function(){
fact_table: '',
lookups: [],
filter_condition:'',
- capacity:'',
+ capacity:'MEDIUM',
"partition_desc" : {
"partition_date_column" : '',
"partition_date_start" : 0,
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c6adb698/webapp/app/partials/cubeDesigner/dimensions.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html b/webapp/app/partials/cubeDesigner/dimensions.html
index 21ef2f5..acb351f 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -125,7 +125,6 @@
</div>
</div>
-
<!-- Edit Dimension Form -->
<script type="text/ng-template" id="addEditDimension.html">
<div class="modal-header">
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c6adb698/webapp/app/partials/cubeDesigner/incremental.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/incremental.html b/webapp/app/partials/cubeDesigner/incremental.html
index 6f40371..6b21dc3 100755
--- a/webapp/app/partials/cubeDesigner/incremental.html
+++ b/webapp/app/partials/cubeDesigner/incremental.html
@@ -46,9 +46,11 @@
ng-required="metaModel.model.partition_desc.partition_date_start"
ng-model="metaModel.model.partition_desc.partition_date_column"
ng-if="state.mode=='edit'"
+ data-placement=""
ng-options="metaModel.model.fact_table+'.'+columns.name as metaModel.model.fact_table+'.'+columns.name for columns in getPartitonColumns(metaModel.model.fact_table)" >
- <option value=""></option>
+ <option value="">--Select Partition Column--</option>
</select>
+ <p class="text-red" ng-if="state.mode=='edit'">(Partition column will select 'date' type column from fact table)</p>
<span ng-if="state.mode=='view'">
{{!!(metaModel.model.partition_desc.partition_date_column)?metaModel.model.partition_desc.partition_date_column: ''}}</span>
</div>
@@ -80,9 +82,8 @@
<div class="row">
<div class="col-xs-12">
<ol class="text-info">
- <li>Leave as default if this cube always need full build</li>
- <li>Please indicate partition column of Fact Table in Hive</li>
- <li>Please indicate start date to just pull certain data from source</li>
+ <li>Not required,leave as default if this cube always need full build</li>
+ <li>If column selected,please indicate start date to just pull certain data from source</li>
</ol>
</div>
</div>
[27/50] incubator-kylin git commit: Merge branch 'master' into staging
Posted by li...@apache.org.
Merge branch 'master' into staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2157fe52
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2157fe52
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2157fe52
Branch: refs/heads/streaming
Commit: 2157fe52a7f2b401c1f3c4742238ee390a996149
Parents: 6074fb4 b6a2bd0
Author: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Authored: Mon Mar 9 11:22:43 2015 +0100
Committer: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Committed: Mon Mar 9 11:22:43 2015 +0100
----------------------------------------------------------------------
jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[38/50] incubator-kylin git commit: Merge pull request #446 from
janzhongi/staging
Posted by li...@apache.org.
Merge pull request #446 from janzhongi/staging
fix filter 0 bytes bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8cca80d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8cca80d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8cca80d5
Branch: refs/heads/streaming
Commit: 8cca80d570c64c71050fad206492b623e43c9d77
Parents: 65445e8 7dae30e
Author: Zhong,Jian <ji...@ebay.com>
Authored: Tue Mar 10 15:27:18 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Tue Mar 10 15:27:18 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/filters/filter.js | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
[25/50] incubator-kylin git commit: KYLIN-625,
extract ICodeSystem to separate query level filter and coprocessor
level filter
Posted by li...@apache.org.
KYLIN-625, extract ICodeSystem to separate query level filter and coprocessor level filter
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/78e9ed04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/78e9ed04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/78e9ed04
Branch: refs/heads/streaming
Commit: 78e9ed04892b8fe6e96432c67720c6999f6b1c35
Parents: fd0fbd5
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Mar 9 18:06:56 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Mar 9 18:06:56 2015 +0800
----------------------------------------------------------------------
.../kylin/metadata/filter/CaseTupleFilter.java | 16 ++---
.../metadata/filter/ColumnTupleFilter.java | 24 ++++---
.../metadata/filter/CompareTupleFilter.java | 71 +++++++-------------
.../metadata/filter/ConstantTupleFilter.java | 22 +++---
.../metadata/filter/DynamicTupleFilter.java | 8 +--
.../metadata/filter/ExtractTupleFilter.java | 18 ++---
.../kylin/metadata/filter/ICodeSystem.java | 23 +++++++
.../metadata/filter/LogicalTupleFilter.java | 26 +++----
.../kylin/metadata/filter/StringCodeSystem.java | 40 +++++++++++
.../kylin/metadata/filter/TupleFilter.java | 8 +--
.../metadata/filter/TupleFilterSerializer.java | 26 +++----
.../kylin/metadata/model/DatabaseDesc.java | 2 +-
.../apache/kylin/metadata/model/TableDesc.java | 3 +-
.../kylin/query/enumerator/CubeEnumerator.java | 16 ++---
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../apache/kylin/query/test/KylinQueryTest.java | 18 +++--
.../storage/filter/BitMapFilterEvaluator.java | 14 ++--
.../kylin/storage/hbase/CubeStorageEngine.java | 5 +-
.../storage/hbase/DerivedFilterTranslator.java | 3 +-
.../hbase/coprocessor/CoprocessorFilter.java | 42 +++++-------
.../hbase/coprocessor/DictCodeSystem.java | 45 +++++++++++++
.../filter/BitMapFilterEvaluatorTest.java | 1 -
.../kylin/storage/filter/FilterBaseTest.java | 4 +-
.../storage/filter/FilterEvaluateTest.java | 36 +++++-----
.../storage/filter/FilterSerializeTest.java | 48 ++++++-------
25 files changed, 304 insertions(+), 217 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
index 5acbe5c..d3ec3e3 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
@@ -33,7 +33,7 @@ public class CaseTupleFilter extends TupleFilter {
private List<TupleFilter> whenFilters;
private List<TupleFilter> thenFilters;
private TupleFilter elseFilter;
- private Collection<String> values;
+ private Collection<?> values;
private int filterIndex;
public CaseTupleFilter() {
@@ -62,16 +62,16 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
if (whenFilters.size() != thenFilters.size()) {
elseFilter = whenFilters.remove(whenFilters.size() - 1);
}
boolean matched = false;
for (int i = 0; i < whenFilters.size(); i++) {
TupleFilter whenFilter = whenFilters.get(i);
- if (whenFilter.evaluate(tuple)) {
+ if (whenFilter.evaluate(tuple, cs)) {
TupleFilter thenFilter = thenFilters.get(i);
- thenFilter.evaluate(tuple);
+ thenFilter.evaluate(tuple, cs);
values = thenFilter.getValues();
matched = true;
break;
@@ -79,7 +79,7 @@ public class CaseTupleFilter extends TupleFilter {
}
if (!matched) {
if (elseFilter != null) {
- elseFilter.evaluate(tuple);
+ elseFilter.evaluate(tuple, cs);
values = elseFilter.getValues();
} else {
values = Collections.emptyList();
@@ -95,17 +95,17 @@ public class CaseTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return this.values;
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index 1cf1f3c..a294112 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -38,12 +38,12 @@ public class ColumnTupleFilter extends TupleFilter {
private TblColRef columnRef;
private Object tupleValue;
- private List<String> values;
+ private List<Object> values;
public ColumnTupleFilter(TblColRef column) {
super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.COLUMN);
this.columnRef = column;
- this.values = new ArrayList<String>(1);
+ this.values = new ArrayList<Object>(1);
this.values.add(null);
}
@@ -66,7 +66,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
this.tupleValue = tuple.getValue(columnRef);
return true;
}
@@ -77,17 +77,20 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
this.values.set(0, (String) this.tupleValue);
return this.values;
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
String table = columnRef.getTable();
BytesUtil.writeUTFString(table, buffer);
+ String columnId = columnRef.getColumn().getId();
+ BytesUtil.writeUTFString(columnId, buffer);
+
String columnName = columnRef.getName();
BytesUtil.writeUTFString(columnName, buffer);
@@ -100,18 +103,21 @@ public class ColumnTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
+ TableDesc table = null;
ColumnDesc column = new ColumnDesc();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
+
String tableName = BytesUtil.readUTFString(buffer);
if (tableName != null) {
- TableDesc table = new TableDesc();
+ table = new TableDesc();
table.setName(tableName);
- column.setTable(table);
}
-
+
+ column.setId(BytesUtil.readUTFString(buffer));
column.setName(BytesUtil.readUTFString(buffer));
column.setDatatype(BytesUtil.readUTFString(buffer));
+ column.init(table);
this.columnRef = new TblColRef(column);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 16114c5..48b31ea 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -34,15 +34,14 @@ import org.apache.kylin.metadata.model.TblColRef;
public class CompareTupleFilter extends TupleFilter {
private TblColRef column;
- private Collection<String> conditionValues;
- private String firstCondValue;
- private Map<String, String> dynamicVariables;
- private String nullString;
+ private Collection<Object> conditionValues;
+ private Object firstCondValue;
+ private Map<String, Object> dynamicVariables;
public CompareTupleFilter(FilterOperatorEnum op) {
super(new ArrayList<TupleFilter>(2), op);
- this.conditionValues = new HashSet<String>();
- this.dynamicVariables = new HashMap<String, String>();
+ this.conditionValues = new HashSet<Object>();
+ this.dynamicVariables = new HashMap<String, Object>();
boolean opGood = (op == FilterOperatorEnum.EQ || op == FilterOperatorEnum.NEQ //
|| op == FilterOperatorEnum.LT || op == FilterOperatorEnum.LTE //
|| op == FilterOperatorEnum.GT || op == FilterOperatorEnum.GTE //
@@ -55,9 +54,9 @@ public class CompareTupleFilter extends TupleFilter {
private CompareTupleFilter(CompareTupleFilter another) {
super(new ArrayList<TupleFilter>(another.children), another.operator);
this.column = another.column;
- this.conditionValues = new HashSet<String>();
+ this.conditionValues = new HashSet<Object>();
this.conditionValues.addAll(another.conditionValues);
- this.dynamicVariables = new HashMap<String, String>();
+ this.dynamicVariables = new HashMap<String, Object>();
this.dynamicVariables.putAll(another.dynamicVariables);
}
@@ -81,10 +80,6 @@ public class CompareTupleFilter extends TupleFilter {
DynamicTupleFilter dynamicFilter = (DynamicTupleFilter) child;
this.dynamicVariables.put(dynamicFilter.getVariableName(), null);
}
- //TODO
- // else if (child instanceof ExtractTupleFilter) {
- // } else if (child instanceof CaseTupleFilter) {
- // }
}
private boolean needSwapOperator() {
@@ -92,11 +87,11 @@ public class CompareTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return conditionValues;
}
- public String getFirstValue() {
+ public Object getFirstValue() {
return firstCondValue;
}
@@ -104,24 +99,16 @@ public class CompareTupleFilter extends TupleFilter {
return column;
}
- public Map<String, String> getVariables() {
+ public Map<String, Object> getVariables() {
return dynamicVariables;
}
- public void bindVariable(String variable, String value) {
+ public void bindVariable(String variable, Object value) {
this.dynamicVariables.put(variable, value);
this.conditionValues.add(value);
this.firstCondValue = this.conditionValues.iterator().next();
}
- public String getNullString() {
- return nullString;
- }
-
- public void setNullString(String nullString) {
- this.nullString = nullString;
- }
-
@Override
public TupleFilter copy() {
return new CompareTupleFilter(this);
@@ -141,34 +128,30 @@ public class CompareTupleFilter extends TupleFilter {
// TODO requires generalize, currently only evaluates COLUMN {op} CONST
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
// extract tuple value
- String tupleValue = null;
+ Object tupleValue = null;
for (TupleFilter filter : this.children) {
if (isConstant(filter) == false) {
- filter.evaluate(tuple);
+ filter.evaluate(tuple, cs);
tupleValue = filter.getValues().iterator().next();
}
}
- // consider null string
- if (nullString != null && nullString.equals(tupleValue)) {
- tupleValue = null;
- }
- if (tupleValue == null) {
+ // consider null case
+ if (cs.isNull(tupleValue)) {
if (operator == FilterOperatorEnum.ISNULL)
return true;
else
return false;
}
-
- // always false if compare to null
- if (firstCondValue.equals(nullString))
+ if (cs.isNull(firstCondValue)) {
return false;
+ }
// tricky here -- order is ensured by string compare (even for number columns)
// because it's row key ID (not real value) being compared
- int comp = tupleValue.compareTo(firstCondValue);
+ int comp = cs.compare(tupleValue, firstCondValue);
boolean result;
switch (operator) {
@@ -212,31 +195,29 @@ public class CompareTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
int size = this.dynamicVariables.size();
BytesUtil.writeVInt(size, buffer);
- for (Map.Entry<String, String> entry : this.dynamicVariables.entrySet()) {
+ for (Map.Entry<String, Object> entry : this.dynamicVariables.entrySet()) {
BytesUtil.writeUTFString(entry.getKey(), buffer);
- BytesUtil.writeUTFString(entry.getValue(), buffer);
+ cs.serialize(entry.getValue(), buffer);
}
- BytesUtil.writeAsciiString(nullString, buffer);
byte[] result = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
return result;
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
this.dynamicVariables.clear();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
for (int i = 0; i < size; i++) {
- String nameString = BytesUtil.readUTFString(buffer);
- String valueString = BytesUtil.readUTFString(buffer);
- bindVariable(nameString, valueString);
+ String name = BytesUtil.readUTFString(buffer);
+ Object value = cs.deserialize(buffer);
+ bindVariable(name, value);
}
- this.nullString = BytesUtil.readAsciiString(buffer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index 1b21728..2280c6a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -35,19 +35,19 @@ public class ConstantTupleFilter extends TupleFilter {
public static final ConstantTupleFilter FALSE = new ConstantTupleFilter();
public static final ConstantTupleFilter TRUE = new ConstantTupleFilter("TRUE");
- private Collection<String> constantValues;
+ private Collection<Object> constantValues;
public ConstantTupleFilter() {
super(Collections.<TupleFilter> emptyList(), FilterOperatorEnum.CONSTANT);
- this.constantValues = new HashSet<String>();
+ this.constantValues = new HashSet<Object>();
}
- public ConstantTupleFilter(String value) {
+ public ConstantTupleFilter(Object value) {
this();
this.constantValues.add(value);
}
- public ConstantTupleFilter(Collection<String> values) {
+ public ConstantTupleFilter(Collection<?> values) {
this();
this.constantValues.addAll(values);
}
@@ -63,7 +63,7 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
return constantValues.size() > 0;
}
@@ -73,17 +73,17 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return this.constantValues;
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
int size = this.constantValues.size();
BytesUtil.writeVInt(size, buffer);
- for (String val : this.constantValues) {
- BytesUtil.writeUTFString(val, buffer);
+ for (Object val : this.constantValues) {
+ cs.serialize(val, buffer);
}
byte[] result = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
@@ -91,12 +91,12 @@ public class ConstantTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
this.constantValues.clear();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
int size = BytesUtil.readVInt(buffer);
for (int i = 0; i < size; i++) {
- this.constantValues.add(BytesUtil.readUTFString(buffer));
+ this.constantValues.add(cs.deserialize(buffer));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
index 08426ba..64f12b8 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/DynamicTupleFilter.java
@@ -53,7 +53,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
return true;
}
@@ -63,12 +63,12 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return Collections.emptyList();
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
BytesUtil.writeUTFString(variableName, buffer);
byte[] result = new byte[buffer.position()];
@@ -77,7 +77,7 @@ public class DynamicTupleFilter extends TupleFilter {
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.wrap(bytes);
this.variableName = BytesUtil.readUTFString(buffer);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
index a02f91a..929ab0f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ExtractTupleFilter.java
@@ -31,12 +31,12 @@ import java.util.List;
public class ExtractTupleFilter extends TupleFilter {
private int date;
- private List<String> values;
+ private List<Object> values;
public ExtractTupleFilter(FilterOperatorEnum op) {
super(new ArrayList<TupleFilter>(3), op);
assert (op == FilterOperatorEnum.EXTRACT);
- this.values = new ArrayList<String>(1);
+ this.values = new ArrayList<Object>(1);
this.values.add(null);
this.date = 0;
}
@@ -52,16 +52,16 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
// extract tuple value
String extractType = null;
String tupleValue = null;
for (TupleFilter filter : this.children) {
- filter.evaluate(tuple);
+ filter.evaluate(tuple, cs);
if (filter instanceof ConstantTupleFilter) {
- tupleValue = filter.getValues().iterator().next();
+ tupleValue = filter.getValues().iterator().next().toString();
} else if (filter instanceof CompareTupleFilter) {
- extractType = filter.getValues().iterator().next();
+ extractType = filter.getValues().iterator().next().toString();
}
}
@@ -106,18 +106,18 @@ public class ExtractTupleFilter extends TupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
this.values.set(0, String.valueOf(this.date));
return this.values;
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java
new file mode 100644
index 0000000..39dcd16
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ICodeSystem.java
@@ -0,0 +1,23 @@
+package org.apache.kylin.metadata.filter;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Decides how constant values in filter are coded and compared.
+ *
+ * TupleFilter are involved in both query engine and coprocessor. In query engine, the values are strings.
+ * In coprocessor, the values are dictionary IDs.
+ *
+ * @author yangli9
+ */
+public interface ICodeSystem {
+
+ boolean isNull(Object value);
+
+ int compare(Object tupleValue, Object constValue);
+
+ void serialize(Object value, ByteBuffer buffer);
+
+ Object deserialize(ByteBuffer buffer);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 2649674..6b77cb5 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -69,43 +69,43 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public boolean evaluate(IEvaluatableTuple tuple) {
+ public boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs) {
switch (this.operator) {
case AND:
- return evalAnd(tuple);
+ return evalAnd(tuple, cs);
case OR:
- return evalOr(tuple);
+ return evalOr(tuple, cs);
case NOT:
- return evalNot(tuple);
+ return evalNot(tuple, cs);
default:
return false;
}
}
- private boolean evalAnd(IEvaluatableTuple tuple) {
+ private boolean evalAnd(IEvaluatableTuple tuple, ICodeSystem cs) {
for (TupleFilter filter : this.children) {
- if (!filter.evaluate(tuple)) {
+ if (!filter.evaluate(tuple, cs)) {
return false;
}
}
return true;
}
- private boolean evalOr(IEvaluatableTuple tuple) {
+ private boolean evalOr(IEvaluatableTuple tuple, ICodeSystem cs) {
for (TupleFilter filter : this.children) {
- if (filter.evaluate(tuple)) {
+ if (filter.evaluate(tuple, cs)) {
return true;
}
}
return false;
}
- private boolean evalNot(IEvaluatableTuple tuple) {
- return !this.children.get(0).evaluate(tuple);
+ private boolean evalNot(IEvaluatableTuple tuple, ICodeSystem cs) {
+ return !this.children.get(0).evaluate(tuple, cs);
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
return Collections.emptyList();
}
@@ -115,12 +115,12 @@ public class LogicalTupleFilter extends TupleFilter {
}
@Override
- public byte[] serialize() {
+ public byte[] serialize(ICodeSystem cs) {
return new byte[0];
}
@Override
- public void deserialize(byte[] bytes) {
+ public void deserialize(byte[] bytes, ICodeSystem cs) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
new file mode 100644
index 0000000..b39b398
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/StringCodeSystem.java
@@ -0,0 +1,40 @@
+package org.apache.kylin.metadata.filter;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.BytesUtil;
+
+/**
+ * A simple code system where all values are strings and conform to string comparison system.
+ *
+ * @author yangli9
+ */
+public class StringCodeSystem implements ICodeSystem {
+
+ public static final StringCodeSystem INSTANCE = new StringCodeSystem();
+
+ private StringCodeSystem() {
+ // singleton
+ }
+
+ @Override
+ public boolean isNull(Object value) {
+ return value == null;
+ }
+
+ @Override
+ public int compare(Object tupleValue, Object constValue) {
+ return ((String) tupleValue).compareTo((String) constValue);
+ }
+
+ @Override
+ public void serialize(Object value, ByteBuffer buffer) {
+ BytesUtil.writeUTFString((String) value, buffer);
+ }
+
+ @Override
+ public Object deserialize(ByteBuffer buffer) {
+ return BytesUtil.readUTFString(buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 9a461c4..b87aae9 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -188,13 +188,13 @@ public abstract class TupleFilter {
public abstract boolean isEvaluable();
- public abstract boolean evaluate(IEvaluatableTuple tuple);
+ public abstract boolean evaluate(IEvaluatableTuple tuple, ICodeSystem cs);
- public abstract Collection<String> getValues();
+ public abstract Collection<?> getValues();
- public abstract byte[] serialize();
+ abstract byte[] serialize(ICodeSystem cs);
- public abstract void deserialize(byte[] bytes);
+ abstract void deserialize(byte[] bytes, ICodeSystem cs);
public static boolean isEvaluableRecursively(TupleFilter filter) {
if (filter == null)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index 04ffeb5..ba4b5e6 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -46,19 +46,19 @@ public class TupleFilterSerializer {
}
}
- public static byte[] serialize(TupleFilter rootFilter) {
- return serialize(rootFilter, null);
+ public static byte[] serialize(TupleFilter rootFilter, ICodeSystem cs) {
+ return serialize(rootFilter, null, cs);
}
- public static byte[] serialize(TupleFilter rootFilter, Decorator decorator) {
+ public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
- internalSerialize(rootFilter, decorator, buffer);
+ internalSerialize(rootFilter, decorator, buffer, cs);
byte[] result = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
return result;
}
- private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer) {
+ private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem cs) {
if (decorator != null) { // give decorator a chance to manipulate the
// output filter
filter = decorator.onSerialize(filter);
@@ -70,24 +70,24 @@ public class TupleFilterSerializer {
if (filter.hasChildren()) {
// serialize filter+true
- serializeFilter(1, filter, decorator, buffer);
+ serializeFilter(1, filter, decorator, buffer, cs);
// serialize children
for (TupleFilter child : filter.getChildren()) {
- internalSerialize(child, decorator, buffer);
+ internalSerialize(child, decorator, buffer, cs);
}
// serialize none
- serializeFilter(-1, filter, decorator, buffer);
+ serializeFilter(-1, filter, decorator, buffer, cs);
} else {
// serialize filter+false
- serializeFilter(0, filter, decorator, buffer);
+ serializeFilter(0, filter, decorator, buffer, cs);
}
}
- private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer) {
+ private static void serializeFilter(int flag, TupleFilter filter, Decorator decorator, ByteBuffer buffer, ICodeSystem cs) {
if (flag < 0) {
BytesUtil.writeVInt(-1, buffer);
} else {
- byte[] bytes = filter.serialize();
+ byte[] bytes = filter.serialize(cs);
int opVal = filter.getOperator().getValue();
BytesUtil.writeVInt(opVal, buffer);
BytesUtil.writeByteArray(bytes, buffer);
@@ -95,7 +95,7 @@ public class TupleFilterSerializer {
}
}
- public static TupleFilter deserialize(byte[] bytes) {
+ public static TupleFilter deserialize(byte[] bytes, ICodeSystem cs) {
ByteBuffer buffer = ByteBuffer.wrap(bytes);
TupleFilter rootFilter = null;
Stack<TupleFilter> parentStack = new Stack<TupleFilter>();
@@ -109,7 +109,7 @@ public class TupleFilterSerializer {
// deserialize filter
TupleFilter filter = createTupleFilter(opVal);
byte[] filetrBytes = BytesUtil.readByteArray(buffer);
- filter.deserialize(filetrBytes);
+ filter.deserialize(filetrBytes, cs);
if (rootFilter == null) {
// push root to stack
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java
index 4b1f739..3269e43 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DatabaseDesc.java
@@ -33,7 +33,7 @@ public class DatabaseDesc {
* @return the name
*/
public String getName() {
- return name.toUpperCase();
+ return name == null ? "null" : name.toUpperCase();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 14ea390..6db1202 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -41,7 +41,7 @@ public class TableDesc extends RootPersistentEntity {
@JsonProperty("columns")
private ColumnDesc[] columns;
- private DatabaseDesc database;
+ private DatabaseDesc database = new DatabaseDesc();
public ColumnDesc findColumnByName(String name) {
//ignore the db name and table name if exists
@@ -111,7 +111,6 @@ public class TableDesc extends RootPersistentEntity {
@JsonProperty("database")
public void setDatabase(String database) {
- this.database = new DatabaseDesc();
this.database.setName(database);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
index dafc960..9edd8cc 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
@@ -18,7 +18,6 @@
package org.apache.kylin.query.enumerator;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -27,17 +26,16 @@ import net.hydromatic.linq4j.Enumerator;
import net.hydromatic.optiq.DataContext;
import net.hydromatic.optiq.jdbc.OptiqConnection;
-import org.eigenbase.reltype.RelDataTypeField;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.storage.IStorageEngine;
-import org.apache.kylin.storage.StorageEngineFactory;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageEngineFactory;
+import org.eigenbase.reltype.RelDataTypeField;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author xjiang
@@ -169,7 +167,7 @@ public class CubeEnumerator implements Enumerator<Object[]> {
if (filter instanceof CompareTupleFilter && optiqContext != null) {
CompareTupleFilter compFilter = (CompareTupleFilter) filter;
- for (Map.Entry<String, String> entry : compFilter.getVariables().entrySet()) {
+ for (Map.Entry<String, Object> entry : compFilter.getVariables().entrySet()) {
String variable = entry.getKey();
Object value = optiqContext.get(variable);
if (value != null) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 0783e31..16f2562 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -156,7 +156,7 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
private CompareTupleFilter mergeToInClause(TupleFilter filter) {
List<? extends TupleFilter> children = filter.getChildren();
TblColRef inColumn = null;
- List<String> inValues = new LinkedList<String>();
+ List<Object> inValues = new LinkedList<Object>();
for (TupleFilter child : children) {
if (child.getOperator() == FilterOperatorEnum.EQ) {
CompareTupleFilter compFilter = (CompareTupleFilter) child;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index ace1dda..fa0ef5b 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -18,11 +18,17 @@
package org.apache.kylin.query.test;
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Properties;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
@@ -35,14 +41,6 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import java.io.File;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.List;
-import java.util.Properties;
-
-import static org.junit.Assert.assertTrue;
-
@Ignore("KylinQueryTest is contained by CombinationTest")
public class KylinQueryTest extends KylinTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java b/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java
index 244662c..6bca948 100644
--- a/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java
@@ -99,22 +99,22 @@ public class BitMapFilterEvaluator {
}
private ConciseSet evalCompareLT(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
return collectRange(filter.getColumn(), null, id - 1);
}
private ConciseSet evalCompareLTE(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
return collectRange(filter.getColumn(), null, id);
}
private ConciseSet evalCompareGT(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
return collectRange(filter.getColumn(), id + 1, null);
}
private ConciseSet evalCompareGTE(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
return collectRange(filter.getColumn(), id, null);
}
@@ -123,7 +123,7 @@ public class BitMapFilterEvaluator {
}
private ConciseSet evalCompareEqual(CompareTupleFilter filter) {
- int id = Dictionary.stringToDictId(filter.getFirstValue());
+ int id = Dictionary.stringToDictId((String) filter.getFirstValue());
ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id, id);
if (bitMap == null)
return null;
@@ -139,8 +139,8 @@ public class BitMapFilterEvaluator {
private ConciseSet evalCompareIn(CompareTupleFilter filter) {
ConciseSet set = new ConciseSet();
- for (String value : filter.getValues()) {
- int id = Dictionary.stringToDictId(value);
+ for (Object value : filter.getValues()) {
+ int id = Dictionary.stringToDictId((String) value);
ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id, id);
if (bitMap == null)
return null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index a4ecc2a..99a7613 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -26,7 +26,6 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -37,6 +36,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -469,7 +469,8 @@ public class CubeStorageEngine implements IStorageEngine {
continue;
}
- ColumnValueRange range = new ColumnValueRange(comp.getColumn(), comp.getValues(), comp.getOperator());
+ @SuppressWarnings("unchecked")
+ ColumnValueRange range = new ColumnValueRange(comp.getColumn(), (Collection<String>) comp.getValues(), comp.getOperator());
andMerge(range, rangeMap);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
index bfa9d3c..7fc0429 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.IEvaluatableTuple;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.StringCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -75,7 +76,7 @@ public class DerivedFilterTranslator {
SingleColumnTuple tuple = new SingleColumnTuple(derivedCol);
for (String[] row : lookup.getAllRows()) {
tuple.value = row[di];
- if (compf.evaluate(tuple)) {
+ if (compf.evaluate(tuple, StringCodeSystem.INSTANCE)) {
collect(row, pi, satisfyingHostRecords);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
index aedce4d..74c7040 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -58,11 +58,12 @@ public class CoprocessorFilter {
}
@Override
+ @SuppressWarnings("unchecked")
public TupleFilter onSerialize(TupleFilter filter) {
if (filter == null)
return null;
- // In case of NOT(unEvaluatableFilter), we should immediatedly replace it as TRUE,
+ // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
// Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter) will
// always return FALSE
if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
@@ -86,29 +87,25 @@ public class CoprocessorFilter {
return filter;
}
- String nullString = nullString(col);
- Collection<String> constValues = compf.getValues();
+ Collection<String> constValues = (Collection<String>) compf.getValues();
if (constValues == null || constValues.isEmpty()) {
- compf.setNullString(nullString); // maybe ISNULL
return filter;
}
TupleFilter result;
CompareTupleFilter newComp = new CompareTupleFilter(compf.getOperator());
- newComp.setNullString(nullString);
newComp.addChild(new ColumnTupleFilter(col));
String v;
- //TODO: seems not working when CompareTupleFilter has multiple values, like IN
- String firstValue = constValues.iterator().next();
// translate constant into rowkey ID
+ String firstValue = constValues.iterator().next();
switch (newComp.getOperator()) {
case EQ:
case IN:
Set<String> newValues = Sets.newHashSet();
for (String value : constValues) {
v = translate(col, value, 0);
- if (!nullString.equals(v))
+ if (!isDictNull(v))
newValues.add(v);
}
if (newValues.isEmpty()) {
@@ -120,7 +117,7 @@ public class CoprocessorFilter {
break;
case NEQ:
v = translate(col, firstValue, 0);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.TRUE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -129,7 +126,7 @@ public class CoprocessorFilter {
break;
case LT:
v = translate(col, firstValue, 1);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.TRUE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -138,7 +135,7 @@ public class CoprocessorFilter {
break;
case LTE:
v = translate(col, firstValue, -1);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.FALSE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -147,7 +144,7 @@ public class CoprocessorFilter {
break;
case GT:
v = translate(col, firstValue, -1);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.TRUE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -156,7 +153,7 @@ public class CoprocessorFilter {
break;
case GTE:
v = translate(col, firstValue, 1);
- if (nullString.equals(v)) {
+ if (isDictNull(v)) {
result = ConstantTupleFilter.FALSE;
} else {
newComp.addChild(new ConstantTupleFilter(v));
@@ -169,12 +166,8 @@ public class CoprocessorFilter {
return result;
}
- private String nullString(TblColRef column) {
- byte[] id = new byte[columnIO.getColumnLength(column)];
- for (int i = 0; i < id.length; i++) {
- id[i] = Dictionary.NULL;
- }
- return Dictionary.dictIdToString(id, 0, id.length);
+ private boolean isDictNull(String v) {
+ return DictCodeSystem.INSTANCE.isNull(v);
}
private String translate(TblColRef column, String v, int roundingFlag) {
@@ -188,17 +181,18 @@ public class CoprocessorFilter {
public static CoprocessorFilter fromFilter(final ISegment seg, TupleFilter rootFilter) {
// translate constants into dictionary IDs via a serialize copy
FilterDecorator filterDecorator = new FilterDecorator(seg);
- byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator);
- TupleFilter copy = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator, DictCodeSystem.INSTANCE);
+ TupleFilter copy = TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
return new CoprocessorFilter(copy, filterDecorator.getUnstrictlyFilteredColumns());
}
public static byte[] serialize(CoprocessorFilter o) {
- return (o.filter == null) ? BytesUtil.EMPTY_BYTE_ARRAY : TupleFilterSerializer.serialize(o.filter);
+ return (o.filter == null) ? BytesUtil.EMPTY_BYTE_ARRAY : TupleFilterSerializer.serialize(o.filter, DictCodeSystem.INSTANCE);
}
public static CoprocessorFilter deserialize(byte[] filterBytes) {
- TupleFilter filter = (filterBytes == null || filterBytes.length == 0) ? null : TupleFilterSerializer.deserialize(filterBytes);
+ TupleFilter filter = (filterBytes == null || filterBytes.length == 0) ? null //
+ : TupleFilterSerializer.deserialize(filterBytes, DictCodeSystem.INSTANCE);
return new CoprocessorFilter(filter, null);
}
@@ -224,7 +218,7 @@ public class CoprocessorFilter {
if (filter == null)
return true;
else
- return filter.evaluate(tuple);
+ return filter.evaluate(tuple, DictCodeSystem.INSTANCE);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
new file mode 100644
index 0000000..14497f4
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
@@ -0,0 +1,45 @@
+package org.apache.kylin.storage.hbase.coprocessor;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.filter.ICodeSystem;
+
+public class DictCodeSystem implements ICodeSystem {
+
+ public static final DictCodeSystem INSTANCE = new DictCodeSystem();
+
+ private DictCodeSystem() {
+ // singleton
+ }
+
+ @Override
+ public boolean isNull(Object value) {
+ if (value == null)
+ return true;
+
+ String v = (String) value;
+ for (int i = 0, n = v.length(); i < n; i++) {
+ if ((byte) v.charAt(i) != Dictionary.NULL)
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int compare(Object tupleValue, Object constValue) {
+ return ((String) tupleValue).compareTo((String) constValue);
+ }
+
+ @Override
+ public void serialize(Object value, ByteBuffer buffer) {
+ BytesUtil.writeUTFString((String) value, buffer);
+ }
+
+ @Override
+ public Object deserialize(ByteBuffer buffer) {
+ return BytesUtil.readUTFString(buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java
index 47b519f..e116aae 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java
@@ -196,7 +196,6 @@ public class BitMapFilterEvaluatorTest {
public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
CompareTupleFilter filter = new CompareTupleFilter(op);
- filter.setNullString(idToStr(Dictionary.NULL_ID[1]));
filter.addChild(columnFilter(col));
for (int i : ids) {
filter.addChild(constFilter(i));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
index 412bab6..97e0e33 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
@@ -36,6 +36,8 @@ import org.apache.kylin.storage.tuple.TupleInfo;
*
*/
public class FilterBaseTest {
+
+ static final ICodeSystem CS = StringCodeSystem.INSTANCE;
protected List<TblColRef> buildGroups() {
List<TblColRef> groups = new ArrayList<TblColRef>();
@@ -200,7 +202,7 @@ public class FilterBaseTest {
protected int evaluateTuples(Collection<Tuple> tuples, TupleFilter filter) {
int match = 0;
for (Tuple t : tuples) {
- if (filter.evaluate(t)) {
+ if (filter.evaluate(t, CS)) {
match++;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java
index feaf98a..19ef2e7 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterEvaluateTest.java
@@ -41,8 +41,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -57,8 +57,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareFilter(groups, 1);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -73,8 +73,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildOrFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -89,8 +89,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildAndFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -105,8 +105,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "0");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -121,8 +121,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "1");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -137,8 +137,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "2");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 1;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -153,8 +153,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "3");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
@@ -169,8 +169,8 @@ public class FilterEvaluateTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "4");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
int number = 10000;
int[] matcheCounts = new int[] { 0, 0, 0 };
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78e9ed04/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
index 07dfc8e..a095012 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterSerializeTest.java
@@ -42,8 +42,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -53,8 +53,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareFilter(groups, 1);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -64,8 +64,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildAndFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -75,8 +75,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildOrFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -90,8 +90,8 @@ public class FilterSerializeTest extends FilterBaseTest {
groups.add(colRef);
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -105,8 +105,8 @@ public class FilterSerializeTest extends FilterBaseTest {
groups.add(colRef);
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -124,8 +124,8 @@ public class FilterSerializeTest extends FilterBaseTest {
groups.add(colRef);
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -142,8 +142,8 @@ public class FilterSerializeTest extends FilterBaseTest {
groups.add(colRef);
TupleFilter filter = buildCompareFilter(groups, 0);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -158,8 +158,8 @@ public class FilterSerializeTest extends FilterBaseTest {
logicFilter.addChild(orFilter);
logicFilter.addChild(andFilter);
- byte[] bytes = TupleFilterSerializer.serialize(logicFilter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(logicFilter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(logicFilter, newFilter);
}
@@ -174,8 +174,8 @@ public class FilterSerializeTest extends FilterBaseTest {
logicFilter.addChild(orFilter);
logicFilter.addChild(andFilter);
- byte[] bytes = TupleFilterSerializer.serialize(logicFilter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(logicFilter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(logicFilter, newFilter);
}
@@ -185,8 +185,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCaseFilter(groups);
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
@@ -196,8 +196,8 @@ public class FilterSerializeTest extends FilterBaseTest {
List<TblColRef> groups = buildGroups();
TupleFilter filter = buildCompareCaseFilter(groups, "0");
- byte[] bytes = TupleFilterSerializer.serialize(filter);
- TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
compareFilter(filter, newFilter);
}
[34/50] incubator-kylin git commit: fix KYLIN-517
Posted by li...@apache.org.
fix KYLIN-517
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ec0fb0f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ec0fb0f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ec0fb0f8
Branch: refs/heads/streaming
Commit: ec0fb0f80e424cfa6983960f861b6c3e69c06c53
Parents: 8dcc9a6
Author: jiazhong <ji...@ebay.com>
Authored: Tue Mar 10 14:43:52 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Tue Mar 10 14:43:52 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +++++++++++---------
webapp/app/less/component.less | 14 +++++--------
.../app/partials/cubeDesigner/data_model.html | 12 +++++------
.../app/partials/cubeDesigner/dimensions.html | 2 +-
5 files changed, 23 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec0fb0f8/webapp/app/js/controllers/cubeDimensions.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeDimensions.js b/webapp/app/js/controllers/cubeDimensions.js
index e57e05b..f323d1f 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -395,7 +395,6 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel) {
if ($scope.state.mode == 'edit') {
$scope.$on('$destroy', function () {
// $scope.dimensionsAdapter();
-
// Emit dimensions edit event in order to re-generate row key.
$scope.$emit('DimensionsEdited');
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec0fb0f8/webapp/app/js/controllers/cubeModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeModel.js b/webapp/app/js/controllers/cubeModel.js
index 1e5fab0..2874742 100644
--- a/webapp/app/js/controllers/cubeModel.js
+++ b/webapp/app/js/controllers/cubeModel.js
@@ -18,7 +18,7 @@
'use strict';
-KylinApp.controller('CubeModelCtrl', function ($scope, $modal,cubeConfig,ModelService,MetaModel,SweetAlert) {
+KylinApp.controller('CubeModelCtrl', function ($scope, $modal,cubeConfig,ModelService,MetaModel,SweetAlert,$log) {
$scope.cubeConfig = cubeConfig;
var DataModel = function () {
@@ -138,17 +138,20 @@ KylinApp.controller('CubeModelCtrl', function ($scope, $modal,cubeConfig,ModelSe
}
};
+
+ $scope.addNewJoin = function(){
+ $scope.newLookup.join.primary_key.push("null");
+ $scope.newLookup.join.foreign_key.push("null");
+ };
+
+ $scope.removeJoin = function($index){
+ $scope.newLookup.join.primary_key.splice($index,1);
+ $scope.newLookup.join.foreign_key.splice($index,1);
+ };
+
$scope.resetParams = function () {
$scope.lookupState.editing = false;
$scope.lookupState.editingIndex = -1;
-
$scope.newLookup = Lookup();
};
-
- // This is for legacy compatibility, assign 'fact_table' property. TODO new cube schema change.
-// $scope.$on('$destroy', function () {
-// if (!$scope.cubeMetaFrame.fact_table) {
-// $scope.cubeMetaFrame.fact_table = $scope.cubeMetaFrame.model.fact_table;
-// }
-// });
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec0fb0f8/webapp/app/less/component.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/component.less b/webapp/app/less/component.less
index c7bd1cf..4caf964 100644
--- a/webapp/app/less/component.less
+++ b/webapp/app/less/component.less
@@ -828,18 +828,14 @@ span.input-icon {
background-color: #C6C41D;
}
-//style fix chosen bug
-#derived .localytics-chosen,#derived .chosen-container{
- width:85% !important;
-}
-#hierarchy .localytics-chosen,#hierarchy .chosen-container {
- width: 100% !important;
-}
-
.sort-item {
padding: 2px;
width: 50px;
height: 20px;
cursor: move;
-// border-left: 1px solid #333;
+}
+
+.cube-dimension .dl-horizontal{
+ margin-top: 0px !important;
+ margin-bottom: 0px !important;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec0fb0f8/webapp/app/partials/cubeDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/data_model.html b/webapp/app/partials/cubeDesigner/data_model.html
index bdfda69..4aeb1da 100644
--- a/webapp/app/partials/cubeDesigner/data_model.html
+++ b/webapp/app/partials/cubeDesigner/data_model.html
@@ -78,7 +78,7 @@
<!-- Join Condition -->
<td>
<ul class="list-unstyled">
- <li ng-repeat="pk in lookup.join.primary_key">
+ <li ng-repeat="pk in lookup.join.primary_key track by $index">
<code>{{lookup.table + '.' + pk}} = {{metaModel.model.fact_table + '.' + lookup.join.foreign_key[$index]}}</code>
</li>
</ul>
@@ -146,11 +146,9 @@
ng-options="columns.name as columns.name for columns in getColumnsByTable(metaModel.model.fact_table)" >
<option value=""></option>
</select>
- <button class="pull-right btn btn-xs btn-danger" style="cursor: pointer" tooltip="Delete Join Condition"
- ng-click="newLookup.join.primary_key.length=newLookup.join.primary_key.length-1;
- removeElement(newLookup.join.primary_key, newLookup.join.primary_key[$index]);
- removeElement(newLookup.join.foreign_key, newLookup.join.foreign_key[$index]);">
- <i class="fa fa-trash-o"></i>
+ <button class="pull-right btn btn-xs btn-danger" style="cursor: pointer" tooltip="Delete"
+ ng-click="removeJoin($index);">
+ <i class="fa fa-trash-o pointer"></i>
</button>
</div>
<div class="space-4"></div>
@@ -163,7 +161,7 @@
<div class="col-sm-3">
<button class="btn btn-xs btn-info"
ng-if="newLookup.join.type"
- ng-click="newLookup.join.primary_key.length=newLookup.join.primary_key.length+1; ">
+ ng-click="addNewJoin();">
<i class="fa fa-plus"></i> New Join Condition</button>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec0fb0f8/webapp/app/partials/cubeDesigner/dimensions.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html b/webapp/app/partials/cubeDesigner/dimensions.html
index acb351f..dbbc46d 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -58,7 +58,7 @@
<th ng-if="state.mode=='edit'">Actions</th>
</tr>
</thead>
- <tbody>
+ <tbody class="cube-dimension">
<tr ng-repeat="dimension in cubeMetaFrame.dimensions | filter:dimState.filter track by $index">
<!--ID -->
<td>
[30/50] incubator-kylin git commit: fix KYLIN-516
Posted by li...@apache.org.
fix KYLIN-516
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8dcc9a63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8dcc9a63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8dcc9a63
Branch: refs/heads/streaming
Commit: 8dcc9a63ec1953e22fc7937513fd42b2e90cb2a1
Parents: 4905a59
Author: jiazhong <ji...@ebay.com>
Authored: Mon Mar 9 23:04:28 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Mar 9 23:04:28 2015 +0800
----------------------------------------------------------------------
webapp/app/partials/cubes/cube_schema.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8dcc9a63/webapp/app/partials/cubes/cube_schema.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cube_schema.html b/webapp/app/partials/cubes/cube_schema.html
index b9359f0..ab9564a 100644
--- a/webapp/app/partials/cubes/cube_schema.html
+++ b/webapp/app/partials/cubes/cube_schema.html
@@ -47,7 +47,7 @@
</div>
</div>
<div class="col-xs-4">
- <button class="btn btn-prev" ng-click="preView()" ng-show="curStep.title!='Cube Info'" ng-disabled="design_form.$invalid">
+ <button class="btn btn-prev" ng-click="preView()" ng-show="curStep.title!='Cube Info'">
<i class="ace-icon fa fa-arrow-left"></i>
Prev
</button>
[05/50] incubator-kylin git commit: Organize Hybrid codes
Posted by li...@apache.org.
Organize Hybrid codes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/13f32602
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/13f32602
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/13f32602
Branch: refs/heads/streaming
Commit: 13f326029ebd70c91d9ae1e953fbef270fb7f983
Parents: 9609c56
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Mar 6 14:25:14 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Mar 6 14:25:14 2015 +0800
----------------------------------------------------------------------
pom.xml | 3 +-
.../apache/kylin/query/routing/RoutingRule.java | 1 -
.../RoutingRules/RealizationSortRule.java | 21 ++----
.../storage/hybrid/HybridStorageEngine.java | 79 ++++++++------------
4 files changed, 41 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/13f32602/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d59cf0c..e99f77d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -579,7 +579,7 @@
<exclue>**/HiveTableReaderTest.java</exclue>
<exclude>**/TableControllerTest.java</exclude>
<exclude>**/Kafka*Test.java</exclude>
- <exclude>org.apache.kylin.streaming.RequesterTest.java</exclude>
+ <exclude>**/RequesterTest.java</exclude>
</excludes>
<systemProperties>
<property>
@@ -618,6 +618,7 @@
<exclude>**/SampleCubeSetupTest.java</exclude>
<exclude>**/KylinQueryTest.java</exclude>
<exclude>**/Kafka*Test.java</exclude>
+ <exclude>**/RequesterTest.java</exclude>
</excludes>
<systemProperties>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/13f32602/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
index da9a081..ff717a2 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
@@ -42,7 +42,6 @@ public abstract class RoutingRule {
//2. exact match prior to week match
static {
rules.add(new RemoveUncapableRealizationsRule());
- // rules.add(new SimpleQueryMoreColumnsCubeFirstRule());
rules.add(new RealizationSortRule());
rules.add(new AdjustForWeaklyMatchedRealization());//this rule might modify olapcontext content, better put it at last
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/13f32602/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
index d0a96b0..ed8644f 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
@@ -21,6 +21,7 @@ package org.apache.kylin.query.routing.RoutingRules;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.RoutingRule;
+import org.apache.kylin.storage.hybrid.HybridInstance;
import java.util.Collections;
import java.util.Comparator;
@@ -51,22 +52,12 @@ public class RealizationSortRule extends RoutingRule {
return comp;
}
- comp = o1.getAllDimensions().size() - o2.getAllDimensions().size();
- if (comp != 0)
- return comp;
-
- comp = o1.getMeasures().size() - o2.getMeasures().size();
-
- if (comp != 0)
- return comp;
-
- long duration1 = o1.getDateRangeEnd() - o1.getDateRangeStart();
- long duration2 = o2.getDateRangeEnd() - o2.getDateRangeStart();
-
- long diff = duration2 - duration1;
-
+ if (o1 instanceof HybridInstance)
+ return -1;
+ else if (o2 instanceof HybridInstance)
+ return 1;
- return diff == 0 ? 0 : (diff > 0) ? 1 : -1;
+ return 0;
}
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/13f32602/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index 3705c63..7688d68 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -1,13 +1,10 @@
package org.apache.kylin.storage.hybrid;
-import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.filter.*;
-import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -18,8 +15,6 @@ import org.apache.kylin.storage.StorageEngineFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
-
/**
* Created by shaoshi on 2/13/15.
*/
@@ -40,54 +35,22 @@ public class HybridStorageEngine implements IStorageEngine {
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest) {
// search the historic realization
-
ITupleIterator iterator1 = searchRealization(hybridInstance.getHistoryRealizationInstance(), context, sqlDigest);
- long boundary = hybridInstance.getHistoryRealizationInstance().getDateRangeEnd();
- FastDateFormat format = FastDateFormat.getInstance("yyyy-MM-dd");
- String boundaryDate = format.format(boundary);
-
- Collection<TblColRef> filterCols = sqlDigest.filterColumns;
-
String modelName = hybridInstance.getModelName();
-
MetadataManager metaMgr = getMetadataManager();
-
DataModelDesc modelDesc = metaMgr.getDataModelDesc(modelName);
+ // if the model isn't partitioned, only query the history
if (modelDesc.getPartitionDesc() == null || modelDesc.getPartitionDesc().getPartitionDateColumnRef() == null)
return iterator1;
- String partitionColFull = modelDesc.getPartitionDesc().getPartitionDateColumn();
-
- String partitionTable = partitionColFull.substring(0, partitionColFull.lastIndexOf("."));
- String partitionCol = partitionColFull.substring(partitionColFull.lastIndexOf(".") + 1);
-
-
- TableDesc factTbl = metaMgr.getTableDesc(partitionTable);
- ColumnDesc columnDesc = factTbl.findColumnByName(partitionCol);
- TblColRef partitionColRef = new TblColRef(columnDesc);
-
-
- // now search the realtime realization, need add the boundary condition
-
-
- CompareTupleFilter compareTupleFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GTE);
- ColumnTupleFilter columnTupleFilter = new ColumnTupleFilter(partitionColRef);
- ConstantTupleFilter constantTupleFilter = new ConstantTupleFilter(boundaryDate);
- compareTupleFilter.addChild(columnTupleFilter);
- compareTupleFilter.addChild(constantTupleFilter);
-
- if (sqlDigest.filter == null) {
- sqlDigest.filter = compareTupleFilter;
- } else {
- LogicalTupleFilter logicalTupleFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ TblColRef partitionColRef = modelDesc.getPartitionDesc().getPartitionDateColumnRef();
- logicalTupleFilter.addChild(sqlDigest.filter);
- logicalTupleFilter.addChild(compareTupleFilter);
+ // add the boundary condition to query real-time
- sqlDigest.filter = logicalTupleFilter;
- }
+ TupleFilter originalFilter = sqlDigest.filter;
+ sqlDigest.filter = createFilterForRealtime(originalFilter, partitionColRef, hybridInstance.getHistoryRealizationInstance().getDateRangeEnd());
boolean addFilterColumn = false, addAllColumn = false;
@@ -101,19 +64,43 @@ public class HybridStorageEngine implements IStorageEngine {
addAllColumn = true;
}
+ // query real-time
ITupleIterator iterator2 = searchRealization(hybridInstance.getRealTimeRealizationInstance(), context, sqlDigest);
// restore the sqlDigest
- sqlDigest.filter = sqlDigest.filter.getChildren().get(0);
+ sqlDigest.filter = originalFilter;
- if(addFilterColumn)
+ if (addFilterColumn)
sqlDigest.filterColumns.remove(partitionColRef);
- if(addAllColumn)
+ if (addAllColumn)
sqlDigest.allColumns.remove(partitionColRef);
+ // combine history and real-time tuple iterator
return new HybridTupleIterator(new ITupleIterator[]{iterator1, iterator2});
-// return new HybridTupleIterator(new ITupleIterator[]{iterator1, ITupleIterator.EMPTY_TUPLE_ITERATOR});
+ }
+
+
+ private TupleFilter createFilterForRealtime(TupleFilter originFilter, TblColRef partitionColRef, long startDate) {
+
+ FastDateFormat format = FastDateFormat.getInstance("yyyy-MM-dd");
+ String boundaryDate = format.format(startDate);
+
+ CompareTupleFilter compareTupleFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GTE);
+ ColumnTupleFilter columnTupleFilter = new ColumnTupleFilter(partitionColRef);
+ ConstantTupleFilter constantTupleFilter = new ConstantTupleFilter(boundaryDate);
+ compareTupleFilter.addChild(columnTupleFilter);
+ compareTupleFilter.addChild(constantTupleFilter);
+
+ if (originFilter == null)
+ return compareTupleFilter;
+
+ LogicalTupleFilter logicalTupleFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+
+ logicalTupleFilter.addChild(originFilter);
+ logicalTupleFilter.addChild(compareTupleFilter);
+
+ return logicalTupleFilter;
}
private ITupleIterator searchRealization(IRealization realization, StorageContext context, SQLDigest sqlDigest) {
[07/50] incubator-kylin git commit: Merge branch 'staging'
Posted by li...@apache.org.
Merge branch 'staging'
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ef27f3a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ef27f3a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ef27f3a1
Branch: refs/heads/streaming
Commit: ef27f3a16c7cd01d073ef7e56f6f13eb27588ac3
Parents: a3b4bcc fcae14e
Author: lukehan <lu...@apache.org>
Authored: Fri Mar 6 14:55:31 2015 +0800
Committer: lukehan <lu...@apache.org>
Committed: Fri Mar 6 14:55:31 2015 +0800
----------------------------------------------------------------------
bin/kylin.sh | 79 ++++
bin/start-kylin.sh | 55 ---
bin/stop-kylin.sh | 21 -
.../model/validation/rule/RowKeyAttrRule.java | 4 +-
docs/website/download/index.md | 19 +-
job/pom.xml | 9 +
.../kylin/metadata/model/PartitionDesc.java | 3 +-
.../kylin/rest/controller/CubeController.java | 8 +-
webapp/app/js/controllers/admin.js | 431 +++++++++----------
webapp/app/js/controllers/cubeAdvanceSetting.js | 8 +
webapp/app/js/controllers/cubeEdit.js | 71 ++-
webapp/app/js/controllers/cubeSchema.js | 57 ++-
webapp/app/js/controllers/cubes.js | 55 +--
webapp/app/js/controllers/page.js | 2 +-
webapp/app/js/controllers/sourceMeta.js | 62 +--
webapp/app/js/model/projectModel.js | 3 +
webapp/app/js/model/tableModel.js | 78 +++-
.../cubeDesigner/advanced_settings.html | 6 +-
.../app/partials/cubeDesigner/incremental.html | 12 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
webapp/app/partials/jobs/job_merge.html | 2 +-
webapp/app/partials/jobs/job_refresh.html | 2 +-
webapp/app/partials/jobs/job_submit.html | 170 ++++----
webapp/app/partials/query/query.html | 4 +-
webapp/app/partials/query/query_detail.html | 17 +-
.../app/partials/tables/source_table_tree.html | 2 +-
26 files changed, 652 insertions(+), 530 deletions(-)
----------------------------------------------------------------------
[28/50] incubator-kylin git commit: fix KYLIN-220
Posted by li...@apache.org.
fix KYLIN-220
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4905a59e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4905a59e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4905a59e
Branch: refs/heads/streaming
Commit: 4905a59ef46eec8b32260440ac3f099660e20969
Parents: f983e6c
Author: jiazhong <ji...@ebay.com>
Authored: Mon Mar 9 18:25:39 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Mar 9 18:25:39 2015 +0800
----------------------------------------------------------------------
webapp/app/less/component.less | 9 +-
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
2 files changed, 81 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4905a59e/webapp/app/less/component.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/component.less b/webapp/app/less/component.less
index 4817d99..c7bd1cf 100644
--- a/webapp/app/less/component.less
+++ b/webapp/app/less/component.less
@@ -835,6 +835,11 @@ span.input-icon {
#hierarchy .localytics-chosen,#hierarchy .chosen-container {
width: 100% !important;
}
-#joinon .localytics-chosen,#joinon .chosen-container {
- width: 45% !important;
+
+.sort-item {
+ padding: 2px;
+ width: 50px;
+ height: 20px;
+ cursor: move;
+// border-left: 1px solid #333;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4905a59e/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index dfa05c6..164ca05 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -91,73 +91,80 @@
<h4 style="margin-left:42px">Rowkeys</h4>
<table style="margin-left:42px; width:92%"
ng-if="cubeMetaFrame.rowkey.rowkey_columns.length > 0"
- class="table table-hover list">
- <tr>
- <th>ID</th>
- <th>Column</th>
- <th>Mandatory</th>
- <th>Dictionary</th>
- <th>Length</th>
- <th ng-if="state.mode=='edit'"></th>
- </tr>
-
- <tr ng-repeat="rowkey_column in cubeMetaFrame.rowkey.rowkey_columns">
- <td>
- <!-- ID -->
- <b>{{($index + 1)}}</b>
- </td>
- <td>
- <!--Column Name -->
- <input type="text" class="form-control" placeholder="Column Name.." ng-if="state.mode=='edit'"
- tooltip="rowkey column name.." tooltip-trigger="focus"
- ng-model="rowkey_column.column" class="form-control">
-
- <span ng-if="state.mode=='view'">{{rowkey_column.column}}</span>
- </td>
- <td>
- <!-- Mandatory -->
- <button type="button " ng-if="state.mode=='edit'"
- class="btn btn-xs btn-default {{rowkey_column.mandatory? 'active':''}}"
- ng-model="rowkey_column.mandatory"
- ng-click="updateMandatory(rowkey_column);"
- btn-checkbox btn-checkbox-true="true" btn-checkbox-false="false">
- {{rowkey_column.mandatory? 'Y':'N'}}
- </button>
-
- <span ng-if="state.mode=='view'">{{rowkey_column.mandatory? 'Y':'N'}}</span>
- </td>
-
- <td>
- <!--Use Dictionary-->
- <div>
- <select ng-if="state.mode=='edit'" style="width:80px;"
- chosen ng-model="rowkey_column.dictionary"
- ng-change="dictionaryUpdated(rowkey_column);"
- ng-options="dt as dt for dt in cubeConfig.dictionaries">
- <option value=""></option>
- </select>
- <span ng-if="state.mode=='view'">{{rowkey_column.dictionary}}</span>
- </div>
- </td>
-
- <td>
- <!--Column Length -->
- <input type="text" class="form-control" placeholder="Column Length.." ng-if="state.mode=='edit'"
- tooltip="rowkey column length.." tooltip-trigger="focus"
- ng-disabled="rowkey_column.dictionary=='true'"
- ng-model="rowkey_column.length" class="form-control">
-
- <span ng-if="state.mode=='view'">{{rowkey_column.length}}</span>
- </td>
-
-
- <td ng-if="state.mode=='edit'">
- <button class="btn btn-xs btn-info"
- ng-click="removeElement(cubeMetaFrame.rowkey.rowkey_columns, rowkey_column)"><i
- class="fa fa-minus"></i>
- </button>
- </td>
- </tr>
+ class="table table-hover list"
+ >
+ <thead>
+ <tr>
+ <th>ID</th>
+ <th>Column</th>
+ <th>Mandatory</th>
+ <th>Dictionary</th>
+ <th>Length</th>
+ <th ng-if="state.mode=='edit'"></th>
+ </tr>
+ </thead>
+
+ <tbody ui-sortable="state.mode=='edit'" ng-model="cubeMetaFrame.rowkey.rowkey_columns">
+
+ <tr ng-repeat="rowkey_column in cubeMetaFrame.rowkey.rowkey_columns" ng-class="state.mode=='edit'?'sort-item':''">
+
+ <td>
+ <!-- ID -->
+ <span class="ng-binding" ng-class="state.mode=='edit'?'badge':''">{{($index + 1)}}</span>
+ </td>
+ <td>
+ <!--Column Name -->
+ <input type="text" class="form-control" placeholder="Column Name.." ng-if="state.mode=='edit'"
+ tooltip="rowkey column name.." tooltip-trigger="focus"
+ ng-model="rowkey_column.column" class="form-control">
+
+ <span ng-if="state.mode=='view'">{{rowkey_column.column}}</span>
+ </td>
+ <td>
+ <!-- Mandatory -->
+ <button type="button " ng-if="state.mode=='edit'"
+ class="btn btn-xs btn-default {{rowkey_column.mandatory? 'active':''}}"
+ ng-model="rowkey_column.mandatory"
+ ng-click="updateMandatory(rowkey_column);"
+ btn-checkbox btn-checkbox-true="true" btn-checkbox-false="false">
+ {{rowkey_column.mandatory? 'Y':'N'}}
+ </button>
+
+ <span ng-if="state.mode=='view'">{{rowkey_column.mandatory? 'Y':'N'}}</span>
+ </td>
+
+ <td>
+ <!--Use Dictionary-->
+ <div>
+ <select ng-if="state.mode=='edit'" style="width:80px;"
+ chosen ng-model="rowkey_column.dictionary"
+ ng-change="dictionaryUpdated(rowkey_column);"
+ ng-options="dt as dt for dt in cubeConfig.dictionaries">
+ <option value=""></option>
+ </select>
+ <span ng-if="state.mode=='view'">{{rowkey_column.dictionary}}</span>
+ </div>
+ </td>
+
+ <td>
+ <!--Column Length -->
+ <input type="text" class="form-control" placeholder="Column Length.." ng-if="state.mode=='edit'"
+ tooltip="rowkey column length.." tooltip-trigger="focus"
+ ng-disabled="rowkey_column.dictionary=='true'"
+ ng-model="rowkey_column.length" class="form-control">
+
+ <span ng-if="state.mode=='view'">{{rowkey_column.length}}</span>
+ </td>
+
+
+ <td ng-if="state.mode=='edit'">
+ <button class="btn btn-xs btn-info"
+ ng-click="removeElement(cubeMetaFrame.rowkey.rowkey_columns, rowkey_column)"><i
+ class="fa fa-minus"></i>
+ </button>
+ </td>
+ </tr>
+ </tbody>
</table>
<button class="btn btn-sm btn-info" style="margin-left:42px"
[33/50] incubator-kylin git commit: add stacktrace to job output when
exception occurs
Posted by li...@apache.org.
add stacktrace to job output when exception occurs
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e80ce69d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e80ce69d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e80ce69d
Branch: refs/heads/streaming
Commit: e80ce69dba08a9acc69a105cf8f175a873228170
Parents: 2157fe5
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Mar 10 11:24:58 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Mar 10 11:24:58 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/execution/AbstractExecutable.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e80ce69d/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index c0c6160..0a230d8 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -33,6 +33,8 @@ import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -81,7 +83,13 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
protected void onExecuteError(Throwable exception, ExecutableContext executableContext) {
if (!isDiscarded()) {
executableManager.addJobInfo(getId(), END_TIME, Long.toString(System.currentTimeMillis()));
- executableManager.updateJobOutput(getId(), ExecutableState.ERROR, null, exception.getLocalizedMessage());
+ String output = null;
+ if (exception != null) {
+ final StringWriter out = new StringWriter();
+ exception.printStackTrace(new PrintWriter(out));
+ output = out.toString();
+ }
+ executableManager.updateJobOutput(getId(), ExecutableState.ERROR, null, output);
} else {
}
}
[46/50] incubator-kylin git commit: Merge branch 'staging' of
https://github.com/KylinOLAP/Kylin into staging
Posted by li...@apache.org.
Merge branch 'staging' of https://github.com/KylinOLAP/Kylin into staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2bbaee6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2bbaee6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2bbaee6c
Branch: refs/heads/streaming
Commit: 2bbaee6c2a665dfd6b02dfde3758890e76a82e5b
Parents: a5821c8 83fc04b
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Mar 11 11:22:04 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Mar 11 11:22:04 2015 +0800
----------------------------------------------------------------------
docs/website/download/index.md | 2 +-
.../org/apache/kylin/jdbc/util/SQLTypeMap.java | 6 +-
.../kylin/job/execution/AbstractExecutable.java | 10 +-
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 -
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/js/listeners.js | 7 -
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
16 files changed, 120 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2bbaee6c/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
[15/50] incubator-kylin git commit: [Kylin-626] Fix JDBC driver when
returning float or double values
Posted by li...@apache.org.
[Kylin-626] Fix JDBC driver when returning float or double values
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/64eeed30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/64eeed30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/64eeed30
Branch: refs/heads/streaming
Commit: 64eeed308a69d8e303696898b609c8d1f18493da
Parents: e6b7e27
Author: Mustangore <mu...@gmail.com>
Authored: Fri Mar 6 14:47:13 2015 +0100
Committer: Mustangore <mu...@gmail.com>
Committed: Fri Mar 6 14:47:13 2015 +0100
----------------------------------------------------------------------
jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/64eeed30/jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java b/jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java
index 07370d7..ec7df0b 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java
@@ -160,10 +160,12 @@ public class SQLTypeMap {
case Types.INTEGER:
return Integer.parseInt(value);
case Types.BIGINT:
- case Types.REAL:
+ return Long.parseLong(value);
case Types.FLOAT:
+ return Float.parseFloat(value);
+ case Types.REAL:
case Types.DOUBLE:
- return Long.parseLong(value);
+ return Double.parseDouble(value);
case Types.BINARY:
case Types.VARBINARY:
case Types.LONGVARBINARY:
[24/50] incubator-kylin git commit: Fix a bug in parsing partition
column
Posted by li...@apache.org.
Fix a bug in parsing partition column
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6074fb4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6074fb4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6074fb4e
Branch: refs/heads/streaming
Commit: 6074fb4e571617919f057053a24e820c57f9de2d
Parents: f020763
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Mon Mar 9 17:33:37 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Mon Mar 9 17:33:37 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/model/CubeDesc.java | 4 ----
.../org/apache/kylin/metadata/model/DataModelDesc.java | 5 +++++
.../org/apache/kylin/metadata/model/PartitionDesc.java | 12 +++++++-----
3 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6074fb4e/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 1f7e02a..0c9cf7a 100644
--- a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -449,10 +449,6 @@ public class CubeDesc extends RootPersistentEntity {
initMeasureReferenceToColumnFamily();
- if (null != this.model.getPartitionDesc()) {
- this.model.getPartitionDesc().init(columnMap);
- }
-
// check all dimension columns are presented on rowkey
List<TblColRef> dimCols = listDimensionColumnsExcludingDerived();
if (rowkey.getRowKeyColumns().length != dimCols.size()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6074fb4e/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 6d1bc65..c9271fc 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -143,8 +143,13 @@ public class DataModelDesc extends RootPersistentEntity {
public void init(Map<String, TableDesc> tables) {
initJoinColumns(tables);
+ initPartitionDesc(tables);
}
+ private void initPartitionDesc(Map<String, TableDesc> tables) {
+ if(this.partitionDesc != null)
+ this.partitionDesc.init(tables);
+ }
private void initJoinColumns(Map<String, TableDesc> tables) {
// join columns may or may not present in cube;
// here we don't modify 'allColumns' and 'dimensionColumns';
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6074fb4e/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 3b3fe94..cf0a8e3 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -47,7 +47,7 @@ public class PartitionDesc {
private TblColRef partitionDateColumnRef;
- public void init(Map<String, Map<String, TblColRef>> columnMap) {
+ public void init(Map<String, TableDesc> tables) {
if (StringUtils.isNotEmpty(partitionDateColumn)) {
partitionDateColumn = partitionDateColumn.toUpperCase();
@@ -55,11 +55,13 @@ public class PartitionDesc {
if (null != columns && columns.length == 3) {
String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();
- Map<String, TblColRef> cols = columnMap.get(tableName);
- if (cols != null) {
- partitionDateColumnRef = cols.get(columns[2].toUpperCase());
+
+ TableDesc table = tables.get(tableName);
+ ColumnDesc col = table.findColumnByName(columns[2]);
+ if (col != null) {
+ partitionDateColumnRef = new TblColRef(col);
} else {
- throw new IllegalStateException("The table '" + tableName + "' provided in 'partition_date_column' doesn't exist.");
+ throw new IllegalStateException("The column '" + partitionDateColumn + "' provided in 'partition_date_column' doesn't exist.");
}
} else {
throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
[21/50] incubator-kylin git commit: KYLIN-627 Hive tables' partition
column was not sync into Kylin
Posted by li...@apache.org.
KYLIN-627 Hive tables' partition column was not sync into Kylin
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/afff4e76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/afff4e76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/afff4e76
Branch: refs/heads/streaming
Commit: afff4e76a8e7a11549efeac57f845694d9f558a2
Parents: cfbd6ae
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Mon Mar 9 17:09:16 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Mon Mar 9 17:09:16 2015 +0800
----------------------------------------------------------------------
.../kylin/metadata/tool/HiveSourceTableLoader.java | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/afff4e76/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java b/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java
index 00891d9..5297188 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java
@@ -81,15 +81,21 @@ public class HiveSourceTableLoader {
for (String tableName : tables) {
Table table = null;
HiveClient hiveClient = new HiveClient();
+ List<FieldSchema> partitionFields = null;
List<FieldSchema> fields = null;
try {
table = hiveClient.getHiveTable(database, tableName);
+ partitionFields = table.getPartitionKeys();
fields = hiveClient.getHiveTableFields(database, tableName);
} catch (Exception e) {
e.printStackTrace();
throw new IOException(e);
}
+ if (fields != null && partitionFields != null && partitionFields.size() > 0) {
+ fields.addAll(partitionFields);
+ }
+
long tableSize = hiveClient.getFileSizeForTable(table);
long tableFileNum = hiveClient.getFileNumberForTable(table);
TableDesc tableDesc = metaMgr.getTableDesc(database + "." + tableName);
@@ -113,12 +119,11 @@ public class HiveSourceTableLoader {
}
tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
- List<FieldSchema> partitionCols = table.getPartitionKeys();
StringBuffer partitionColumnString = new StringBuffer();
- for (int i = 0, n = partitionCols.size(); i < n; i++) {
+ for (int i = 0, n = partitionFields.size(); i < n; i++) {
if (i > 0)
partitionColumnString.append(", ");
- partitionColumnString.append(partitionCols.get(i).getName().toUpperCase());
+ partitionColumnString.append(partitionFields.get(i).getName().toUpperCase());
}
Map<String, String> map = metaMgr.getTableDescExd(tableDesc.getIdentity());
@@ -135,7 +140,7 @@ public class HiveSourceTableLoader {
map.put(MetadataConstants.TABLE_EXD_PC, partitionColumnString.toString());
map.put(MetadataConstants.TABLE_EXD_TFS, String.valueOf(tableSize));
map.put(MetadataConstants.TABLE_EXD_TNF, String.valueOf(tableFileNum));
- map.put(MetadataConstants.TABLE_EXD_PARTITIONED, Boolean.valueOf(partitionCols != null && partitionCols.size()>0).toString());
+ map.put(MetadataConstants.TABLE_EXD_PARTITIONED, Boolean.valueOf(partitionFields != null && partitionFields.size() > 0).toString());
metaMgr.saveSourceTable(tableDesc);
metaMgr.saveTableExd(tableDesc.getIdentity(), map);
[40/50] incubator-kylin git commit: Merge pull request #447 from
janzhongi/staging
Posted by li...@apache.org.
Merge pull request #447 from janzhongi/staging
config right authorize url Kylin-570
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5f9e2c9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5f9e2c9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5f9e2c9e
Branch: refs/heads/streaming
Commit: 5f9e2c9eb49a00331297658bfb50f02dfeb4fb46
Parents: 8cca80d 2e98153
Author: Zhong,Jian <ji...@ebay.com>
Authored: Tue Mar 10 17:18:54 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Tue Mar 10 17:18:54 2015 +0800
----------------------------------------------------------------------
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 --
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/listeners.js | 7 -------
4 files changed, 2 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
[35/50] incubator-kylin git commit: Merge pull request #445 from
janzhongi/staging
Posted by li...@apache.org.
Merge pull request #445 from janzhongi/staging
fix UI issues when create&edit cube, KYLIN-220,KYLIN-516,KYLIN-517
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/65445e82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/65445e82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/65445e82
Branch: refs/heads/streaming
Commit: 65445e82833120e52f1af6aaaf2fe6464bb4cb3e
Parents: e80ce69 ec0fb0f
Author: Zhong,Jian <ji...@ebay.com>
Authored: Tue Mar 10 14:49:36 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Tue Mar 10 14:49:36 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
7 files changed, 102 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
[16/50] incubator-kylin git commit: GTStore interface updated for row
block index
Posted by li...@apache.org.
GTStore interface updated for row block index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9ec330a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9ec330a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9ec330a1
Branch: refs/heads/streaming
Commit: 9ec330a1fd7214b06eff40de719c52bd0652d8e1
Parents: cebff40
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Mar 9 14:49:34 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Mar 9 14:49:34 2015 +0800
----------------------------------------------------------------------
.../kylin/storage/gridtable/GTBuilder.java | 4 ++--
.../kylin/storage/gridtable/GTRowBlock.java | 1 +
.../kylin/storage/gridtable/GTRowBlockIndex.java | 5 +++++
.../apache/kylin/storage/gridtable/GTStore.java | 19 +++++++++++++++----
4 files changed, 23 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9ec330a1/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
index 9e29b52..0083b86 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
@@ -5,12 +5,12 @@ import java.io.Flushable;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.kylin.storage.gridtable.GTStore.GTBlockWriter;
+import org.apache.kylin.storage.gridtable.GTStore.GTWriter;
public class GTBuilder implements Closeable, Flushable {
final private GTInfo info;
- final private GTBlockWriter writer;
+ final private GTWriter writer;
private GTRowBlock block;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9ec330a1/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
index 4526ef4..6db08ab 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
@@ -7,6 +7,7 @@ class GTRowBlock {
private static final int PRIMARY_KEY_CAPACITY = 2048;
private static final int CELL_BLOCK_CAPACITY = 128 * 1024;
+ int seqId; // 0, 1, 2...
int nRows;
ByteBuffer primaryKey; // the primary key of the first row
ByteBuffer[] cellBlocks; // cells for each column block
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9ec330a1/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlockIndex.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlockIndex.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlockIndex.java
new file mode 100644
index 0000000..21b0a9a
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlockIndex.java
@@ -0,0 +1,5 @@
+package org.apache.kylin.storage.gridtable;
+
+public class GTRowBlockIndex {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9ec330a1/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
index 3ccaa00..b665fd6 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
@@ -1,5 +1,7 @@
package org.apache.kylin.storage.gridtable;
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
import java.io.Closeable;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -11,14 +13,23 @@ public interface GTStore {
public String getStorageDescription();
- public GTBlockWriter rebuild(int shard);
+ // ============================================================================
+
+ public GTWriter rebuild(int shard);
- public GTBlockScanner scan(ByteBuffer pkStart, ByteBuffer pkEndExclusive, int[] colBlocks);
+ public GTScanner scan(ByteBuffer pkStart, ByteBuffer pkEndExclusive, ConciseSet selectedRowBlcoks, int[] selectedColBlocks);
- public interface GTBlockWriter extends Closeable {
+ public interface GTWriter extends Closeable {
void write(GTRowBlock block) throws IOException;
}
- public interface GTBlockScanner extends Iterator<GTRowBlock>, Closeable {
+ public interface GTScanner extends Iterator<GTRowBlock>, Closeable {
}
+
+ // ============================================================================
+
+ public void saveRowBlockIndex(int col, GTRowBlockIndex index);
+
+ public GTRowBlockIndex loadRowBlockIndex(int col);
+
}
[42/50] incubator-kylin git commit: Merge pull request #448 from
janzhongi/streaming
Posted by li...@apache.org.
Merge pull request #448 from janzhongi/streaming
merge staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b25e227b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b25e227b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b25e227b
Branch: refs/heads/streaming
Commit: b25e227be92ed437d4ddafac40ae4da44579de3a
Parents: bb83ca1 e3c5072
Author: Zhong,Jian <ji...@ebay.com>
Authored: Tue Mar 10 18:22:07 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Tue Mar 10 18:22:07 2015 +0800
----------------------------------------------------------------------
.../kylin/job/execution/AbstractExecutable.java | 10 +-
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 -
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/js/listeners.js | 7 -
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
14 files changed, 115 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
[39/50] incubator-kylin git commit: config right authorize url
Kylin-570
Posted by li...@apache.org.
config right authorize url Kylin-570
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2e981533
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2e981533
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2e981533
Branch: refs/heads/streaming
Commit: 2e981533a024d4b9e5ee7f1d8132ed5ca16b5563
Parents: 7dae30e
Author: jiazhong <ji...@ebay.com>
Authored: Tue Mar 10 17:17:52 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Tue Mar 10 17:17:52 2015 +0800
----------------------------------------------------------------------
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 --
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/listeners.js | 7 -------
4 files changed, 2 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2e981533/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index af1b83b..518a9bd 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -12,7 +12,7 @@
<scr:http auto-config="true" use-expressions="true">
<scr:http-basic entry-point-ref="unauthorisedEntryPoint" />
- <scr:intercept-url pattern="/api/user/authenticate*/**" access="permitAll" />
+ <scr:intercept-url pattern="/api/user/authentication*/**" access="permitAll" />
<scr:intercept-url pattern="/api/query*/**" access="isAuthenticated()" />
<scr:intercept-url pattern="/api/metadata*/**" access="isAuthenticated()" />
<scr:intercept-url pattern="/api/**/metrics" access="permitAll" />
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2e981533/webapp/app/js/controllers/auth.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/auth.js b/webapp/app/js/controllers/auth.js
index 376d1b1..f7fbfea 100644
--- a/webapp/app/js/controllers/auth.js
+++ b/webapp/app/js/controllers/auth.js
@@ -43,7 +43,5 @@ KylinApp.controller('LoginCtrl', function ($scope, $rootScope, $location, $base6
$scope.loading = false;
$scope.error = "Unable to login, please check your username/password.";
});
-
- console.debug("Login event requested.");
};
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2e981533/webapp/app/js/controllers/cube.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cube.js b/webapp/app/js/controllers/cube.js
index 689496a..607706d 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -18,7 +18,7 @@
'use strict';
-KylinApp.controller('CubeCtrl', function ($scope, AccessService, MessageService, CubeService, TableService, CubeGraphService, UserService, AuthenticationService,SweetAlert) {
+KylinApp.controller('CubeCtrl', function ($scope, AccessService, MessageService, CubeService, TableService, CubeGraphService, UserService,SweetAlert) {
$scope.newAccess = null;
$scope.state = {jsonEdit: false};
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2e981533/webapp/app/js/listeners.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/listeners.js b/webapp/app/js/listeners.js
index 45f886a..4524696 100644
--- a/webapp/app/js/listeners.js
+++ b/webapp/app/js/listeners.js
@@ -92,11 +92,4 @@ KylinApp.run(function ($rootScope,$http, $location, UserService, AuthenticationS
});
- /**
- * On 'event:error', resend all the 500 requests.
- */
-// $rootScope.$on('event:error', function (event, message) {
-// var msg = !!(message) ? message : 'Failed to take action.';
-// SweetAlert.swal('Oops...', 'Action Failed: ' + msg, 'error');
-// });
});
[49/50] incubator-kylin git commit: minor change to kafka test
Posted by li...@apache.org.
minor change to kafka test
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0b13251b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0b13251b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0b13251b
Branch: refs/heads/streaming
Commit: 0b13251b57b426bbea54f6037f9a09a0778951d0
Parents: 0b71528
Author: honma <ho...@ebay.com>
Authored: Wed Mar 11 00:21:59 2015 -0700
Committer: honma <ho...@ebay.com>
Committed: Wed Mar 11 15:30:52 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/streaming/KafkaConsumer.java | 4 +++-
.../test/java/org/apache/kylin/streaming/KafkaConsumerTest.java | 2 ++
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0b13251b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
index a9f5679..60f85d8 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
@@ -112,10 +112,12 @@ public class KafkaConsumer implements Runnable {
}
for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(topic, partitionId)) {
final ByteBuffer payload = messageAndOffset.message().payload();
+
//TODO use ByteBuffer maybe
byte[] bytes = new byte[payload.limit()];
payload.get(bytes);
- logger.debug("get message offset:" + messageAndOffset.offset());
+ logger.info("get message offset:" + messageAndOffset.offset());
+ logger.info(new String(bytes));
try {
streamQueue.put(new Stream(messageAndOffset.offset(), bytes));
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0b13251b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
index 91e06fc..a6176ab 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
@@ -92,6 +92,8 @@ public class KafkaConsumerTest extends KafkaBaseTest {
for (BlockingQueue<Stream> queue : queues) {
count += queue.size();
}
+
+ logger.info("count of messages are " + count);
//since there will be historical data
assertTrue(count >= TOTAL_SEND_COUNT);
}
[45/50] incubator-kylin git commit: KYLIN-629 Kylin failed to run
mapreduce job if there is no mapreduce.application.classpath in
mapred-site.xml
Posted by li...@apache.org.
KYLIN-629 Kylin failed to run mapreduce job if there is no mapreduce.application.classpath in mapred-site.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a5821c83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a5821c83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a5821c83
Branch: refs/heads/streaming
Commit: a5821c83f8393853dba271522c2c96bd912f0b0e
Parents: 93f1f26
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Mar 11 11:21:26 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Mar 11 11:21:26 2015 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/AbstractHadoopJob.java | 64 ++++++++++++++------
1 file changed, 44 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a5821c83/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index ff19357..087dd04 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -23,18 +23,6 @@ package org.apache.kylin.job.hadoop;
*
*/
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.StringSplitter;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.job.exception.JobException;
-import org.apache.kylin.job.tools.OptionsHelper;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -50,9 +38,22 @@ import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.JobInstance;
+import org.apache.kylin.job.cmd.ShellCmdOutput;
+import org.apache.kylin.job.exception.JobException;
+import org.apache.kylin.job.tools.OptionsHelper;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,6 +64,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import static org.apache.hadoop.util.StringUtils.formatTime;
+
@SuppressWarnings("static-access")
public abstract class AbstractHadoopJob extends Configured implements Tool {
protected static final Logger logger = LoggerFactory.getLogger(AbstractHadoopJob.class);
@@ -121,7 +124,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
} else {
job.waitForCompletion(true);
retVal = job.isSuccessful() ? 0 : 1;
- logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + StringUtils.formatTime((System.nanoTime() - start) / 1000000L));
+ logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + formatTime((System.nanoTime() - start) / 1000000L));
}
return retVal;
}
@@ -135,7 +138,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
System.exit(5);
}
}
-
+
private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
protected void setJobClasspath(Job job) {
@@ -154,16 +157,37 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// yarn classpath is comma separated
kylinHiveDependency = kylinHiveDependency.replace(":", ",");
Configuration jobConf = job.getConfiguration();
- final String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
- if (classpath == null) {
- jobConf.set(MAP_REDUCE_CLASSPATH, kylinHiveDependency);
- } else {
- jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
+ String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
+ if (classpath == null || classpath.length() == 0) {
+ logger.info("Didn't find " + MAP_REDUCE_CLASSPATH + " in job configuration, will run 'mapred classpath' to get the default value.");
+ classpath = getDefaultMapRedClasspath();
+ logger.info("The default mapred classpath is: " + classpath);
}
+
+ jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
+
}
logger.info("Hadoop job classpath is: " + job.getConfiguration().get(MAP_REDUCE_CLASSPATH));
}
+
+ private String getDefaultMapRedClasspath() {
+
+ String classpath = "";
+ try {
+ CliCommandExecutor executor = KylinConfig.getInstanceFromEnv().getCliCommandExecutor();
+ ShellCmdOutput output = new ShellCmdOutput();
+ executor.execute("mapred classpath", output);
+
+ classpath = output.getOutput();
+ } catch (IOException e) {
+ logger.error("Failed to run: 'mapred classpath'.", e);
+ }
+
+ return classpath;
+ }
+
+
public void addInputDirs(String input, Job job) throws IOException {
for (String inp : StringSplitter.split(input, ",")) {
inp = inp.trim();
[44/50] incubator-kylin git commit: #KYLIN-627, #KYLIN_628,
update binary package with bug fixes
Posted by li...@apache.org.
#KYLIN-627, #KYLIN_628, update binary package with bug fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/83fc04ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/83fc04ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/83fc04ba
Branch: refs/heads/streaming
Commit: 83fc04ba4da161b9e33aa6e8df9651f415498d2c
Parents: 59a1d66
Author: lukehan <lu...@apache.org>
Authored: Tue Mar 10 09:51:32 2015 -0700
Committer: lukehan <lu...@apache.org>
Committed: Tue Mar 10 09:51:32 2015 -0700
----------------------------------------------------------------------
docs/website/download/index.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/83fc04ba/docs/website/download/index.md
----------------------------------------------------------------------
diff --git a/docs/website/download/index.md b/docs/website/download/index.md
index 170557f..4fc10c8 100644
--- a/docs/website/download/index.md
+++ b/docs/website/download/index.md
@@ -15,7 +15,7 @@ title: Community
<p class="big_text">Binary Package (v0.7.x)</p>
<p><a href="http://kylin.incubator.apache.org/download/kylin-0.7.1-SNAPSHOT.tgz" target="_blank">v0.7.1-SNAPSHOT</a>
- <br/>Updated Date - March 6, 2015 <br/>
+ <br/>Updated Date - March 9, 2015 <br/>
</p>
</div>
[32/50] incubator-kylin git commit: Merge branch 'staging' of
https://github.com/KylinOLAP/Kylin into staging
Posted by li...@apache.org.
Merge branch 'staging' of https://github.com/KylinOLAP/Kylin into staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/76e5fd3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/76e5fd3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/76e5fd3e
Branch: refs/heads/streaming
Commit: 76e5fd3e7ada79046ee7ee2b844e5dfdbbfbeaf5
Parents: 30991df 2157fe5
Author: lukehan <lu...@apache.org>
Authored: Mon Mar 9 17:25:28 2015 -0700
Committer: lukehan <lu...@apache.org>
Committed: Mon Mar 9 17:25:28 2015 -0700
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 4 -
.../org/apache/kylin/jdbc/util/SQLTypeMap.java | 6 +-
.../kylin/metadata/model/DataModelDesc.java | 5 ++
.../kylin/metadata/model/PartitionDesc.java | 12 +--
.../metadata/tool/HiveSourceTableLoader.java | 13 ++-
webapp/app/js/controllers/cubeEdit.js | 90 +++++++++++++-------
webapp/app/js/controllers/cubeModel.js | 35 ++++++--
webapp/app/js/controllers/cubeSchema.js | 7 --
webapp/app/js/model/jobListModel.js | 11 ---
webapp/app/js/model/metaModel.js | 2 +-
.../cubeDesigner/advanced_settings.html | 1 +
.../app/partials/cubeDesigner/dimensions.html | 1 -
.../app/partials/cubeDesigner/incremental.html | 10 ++-
13 files changed, 121 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
[12/50] incubator-kylin git commit: Merge branch 'streaming' of
https://github.com/KylinOLAP/Kylin into streaming
Posted by li...@apache.org.
Merge branch 'streaming' of https://github.com/KylinOLAP/Kylin into streaming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cebff400
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cebff400
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cebff400
Branch: refs/heads/streaming
Commit: cebff400a0ac9e9b57a9ea3b2a4e72efccd6baf7
Parents: b093bd5 c3c47db
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 6 18:16:34 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 6 18:16:34 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 11 +++
.../org/apache/kylin/cube/CubeSegmentsTest.java | 4 +-
.../apache/kylin/cube/cuboid/CuboidTest.java | 13 ---
.../apache/kylin/dict/HiveTableReaderTest.java | 2 +-
.../hybrid/test_kylin_hybrid_left_join.json | 4 +-
.../invertedindex/test_kylin_ii_left_join.json | 4 +-
.../test_kylin_ii_left_join_desc.json | 4 +-
.../test_kylin_inner_join_model_desc.json | 59 +++++++++++++
.../test_kylin_left_join_model_desc.json | 59 +++++++++++++
.../invertedindex/index/TableRecordInfo.java | 30 -------
.../kylin/invertedindex/model/IIDesc.java | 19 ++--
.../kylin/invertedindex/model/IIDimension.java | 69 ---------------
.../invertedindex/IIDescManagerTest.java | 2 +-
.../invertedindex/invertedindex/IIDescTest.java | 4 +-
.../invertedindex/IIInstanceTest.java | 4 +-
.../invertedindex/InvertedIndexLocalTest.java | 2 +-
.../kylin/job/BuildCubeWithEngineTest.java | 22 ++---
.../apache/kylin/job/BuildIIWithEngineTest.java | 93 +++++++++++---------
.../kylin/job/dataGen/FactTableGenerator.java | 2 -
.../kylin/metadata/model/DataModelDesc.java | 20 ++++-
.../kylin/metadata/model/DimensionDesc.java | 69 +++++++++++++++
.../kylin/metadata/project/ProjectL2Cache.java | 3 +
.../realization/RealizationRegistry.java | 1 +
.../kylin/metadata/MetadataManagerTest.java | 19 ++--
pom.xml | 4 +
.../kylin/query/enumerator/CubeEnumerator.java | 15 +++-
.../apache/kylin/query/routing/RoutingRule.java | 1 -
.../RoutingRules/RealizationSortRule.java | 20 ++---
.../kylin/query/test/CombinationTest.java | 13 ++-
.../apache/kylin/query/test/IIQueryTest.java | 28 ++++--
.../apache/kylin/query/test/KylinQueryTest.java | 55 ++++--------
.../apache/kylin/query/test/KylinTestBase.java | 2 +-
query/src/test/resources/query/sql/query09.sql | 8 +-
query/src/test/resources/query/sql/query10.sql | 8 +-
query/src/test/resources/query/sql/query11.sql | 8 +-
query/src/test/resources/query/sql/query12.sql | 8 +-
query/src/test/resources/query/sql/query13.sql | 8 +-
query/src/test/resources/query/sql/query14.sql | 11 +--
query/src/test/resources/query/sql/query15.sql | 10 ++-
query/src/test/resources/query/sql/query16.sql | 11 +--
query/src/test/resources/query/sql/query17.sql | 10 ++-
query/src/test/resources/query/sql/query18.sql | 10 ++-
query/src/test/resources/query/sql/query19.sql | 10 ++-
query/src/test/resources/query/sql/query20.sql | 10 ++-
query/src/test/resources/query/sql/query42.sql | 8 +-
query/src/test/resources/query/sql/query43.sql | 8 +-
query/src/test/resources/query/sql/query44.sql | 9 +-
query/src/test/resources/query/sql/query50.sql | 8 +-
query/src/test/resources/query/sql/query51.sql | 8 +-
query/src/test/resources/query/sql/query54.sql | 8 +-
query/src/test/resources/query/sql/query56.sql | 6 +-
query/src/test/resources/query/sql/query57.sql | 6 +-
query/src/test/resources/query/sql/query58.sql | 6 +-
query/src/test/resources/query/sql/query60.sql | 7 +-
query/src/test/resources/query/sql/query61.sql | 7 +-
query/src/test/resources/query/sql/query66.sql | 8 +-
query/src/test/resources/query/sql/query72.sql | 8 +-
.../resources/query/sql_derived/query05.sql | 7 +-
.../resources/query/sql_orderby/query01.sql | 8 +-
.../resources/query/sql_subquery/query00.sql | 7 +-
.../resources/query/sql_subquery/query01.sql | 7 +-
.../resources/query/sql_subquery/query03.sql | 15 +++-
.../resources/query/sql_subquery/query04.sql | 15 +++-
.../resources/query/sql_subquery/query05.sql | 9 ++
.../kylin/rest/service/CacheServiceTest.java | 6 +-
.../kylin/storage/hybrid/HybridInstance.java | 4 +
.../storage/hybrid/HybridStorageEngine.java | 79 +++++++----------
.../storage/hbase/InvertedIndexHBaseTest.java | 2 +-
.../endpoint/EndpointAggregationTest.java | 2 +-
.../endpoint/TableRecordInfoTest.java | 2 +-
.../kylin/storage/hybrid/HybridManagerTest.java | 2 +-
.../apache/kylin/storage/test/StorageTest.java | 2 +-
72 files changed, 629 insertions(+), 404 deletions(-)
----------------------------------------------------------------------
[22/50] incubator-kylin git commit: Merge pull request #443 from
janzhongi/staging
Posted by li...@apache.org.
Merge pull request #443 from janzhongi/staging
support 'string' and 'date' type column as options for partition column
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a9bf9a34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a9bf9a34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a9bf9a34
Branch: refs/heads/streaming
Commit: a9bf9a34c59118f3712cfae8df7bf82e0a6e7814
Parents: 4be14bb f983e6c
Author: Zhong,Jian <ji...@ebay.com>
Authored: Mon Mar 9 17:10:09 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Mon Mar 9 17:10:09 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 2 +-
webapp/app/partials/cubeDesigner/incremental.html | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[04/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c9147ca4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c9147ca4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c9147ca4
Branch: refs/heads/streaming
Commit: c9147ca40b6ad66e9e7558e887445df36f579cbd
Parents: 4a59d38
Author: qianhao.zhou <qi...@ebay.com>
Authored: Fri Mar 6 14:22:36 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Fri Mar 6 14:22:36 2015 +0800
----------------------------------------------------------------------
bin/kylin.sh | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c9147ca4/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
index 4efaf9e..7981bb8 100644
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -54,6 +54,7 @@ then
echo "Please visit http://<ip>:7070/kylin"
fi
echo "You can check the log at ${tomcat_root}/logs/kylin.log"
+ exit 0
elif [ $1 == "stop" ]
then
if [ ! -f "${KYLIN_HOME}/pid" ]
@@ -71,6 +72,8 @@ then
kill $pid
fi
rm ${KYLIN_HOME}/pid
+ exit 0
else
echo "usage: kylin.sh start or kylin.sh stop"
+ exit 1
fi
\ No newline at end of file
[11/50] incubator-kylin git commit: KYLIN-625, initial skeleton
Posted by li...@apache.org.
KYLIN-625, initial skeleton
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b093bd5e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b093bd5e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b093bd5e
Branch: refs/heads/streaming
Commit: b093bd5e2dd4d2c14b2d0462194f0cdafc923a7e
Parents: 0accec1
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 6 18:16:11 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 6 18:16:11 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 18 +++++
.../metadata/measure/BigDecimalSerializer.java | 18 ++++-
.../metadata/measure/DataTypeSerializer.java | 83 ++++++++++++++++++++
.../metadata/measure/DoubleSerializer.java | 7 +-
.../kylin/metadata/measure/HLLCSerializer.java | 7 +-
.../kylin/metadata/measure/LongSerializer.java | 14 +++-
.../kylin/metadata/measure/MeasureCodec.java | 8 +-
.../metadata/measure/MeasureSerializer.java | 68 ----------------
.../kylin/storage/gridtable/GTBuilder.java | 68 ++++++++++++++++
.../apache/kylin/storage/gridtable/GTInfo.java | 52 ++++++++++++
.../kylin/storage/gridtable/GTRecord.java | 15 ++++
.../kylin/storage/gridtable/GTRowBlock.java | 33 ++++++++
.../apache/kylin/storage/gridtable/GTStore.java | 24 ++++++
.../kylin/storage/gridtable/GridTable.java | 19 +++++
.../kylin/storage/gridtable/IKVStoreReader.java | 11 +++
.../kylin/storage/gridtable/IKVStoreWriter.java | 11 +++
16 files changed, 377 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 381dea6..686321b 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -214,6 +214,24 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
in.get(registers);
}
}
+
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ int len;
+
+ byte scheme = in.get();
+ if (scheme == 0) { // map scheme
+ int size = BytesUtil.readVInt(in);
+ int indexLen = getRegisterIndexSize();
+ len = in.position() - mark + (indexLen + 1) * size;
+ } else {
+ len = in.position() - mark + m;
+ }
+
+ in.position(mark);
+ return len;
+
+ }
public void writeRegistersArray(final ByteBuffer out) {
out.put(this.registers);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
index 2eab9e4..78654ea 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
@@ -23,14 +23,13 @@ import java.math.BigInteger;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.util.Bytes;
-
import org.apache.kylin.common.util.BytesUtil;
/**
* @author yangli9
*
*/
-public class BigDecimalSerializer extends MeasureSerializer<BigDecimal> {
+public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
@Override
public void serialize(BigDecimal value, ByteBuffer out) {
@@ -52,6 +51,20 @@ public class BigDecimalSerializer extends MeasureSerializer<BigDecimal> {
return new BigDecimal(new BigInteger(bytes), scale);
}
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ @SuppressWarnings("unused")
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+ int len = in.position() - mark + n;
+
+ in.position(mark);
+ return len;
+ }
+
@Override
public BigDecimal valueOf(byte[] value) {
if (value == null)
@@ -59,5 +72,4 @@ public class BigDecimalSerializer extends MeasureSerializer<BigDecimal> {
else
return new BigDecimal(Bytes.toString(value));
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java
new file mode 100644
index 0000000..ecaaea7
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.measure;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.metadata.model.DataType;
+
+/**
+ * @author yangli9
+ *
+ */
+abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
+
+ final static HashMap<String, Class<?>> implementations = new HashMap<String, Class<?>>();
+ static {
+ implementations.put("decimal", BigDecimalSerializer.class);
+ implementations.put("double", DoubleSerializer.class);
+ implementations.put("float", DoubleSerializer.class);
+ implementations.put("bigint", LongSerializer.class);
+ implementations.put("long", LongSerializer.class);
+ implementations.put("integer", LongSerializer.class);
+ implementations.put("int", LongSerializer.class);
+ }
+
+ public static DataTypeSerializer<?> create(String dataType) {
+ DataType type = DataType.getInstance(dataType);
+ if (type.isHLLC()) {
+ return new HLLCSerializer(type.getPrecision());
+ }
+
+ Class<?> clz = implementations.get(type.getName());
+ if (clz == null)
+ throw new RuntimeException("No MeasureSerializer for type " + dataType);
+
+ try {
+ return (DataTypeSerializer<?>) clz.newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** peek into buffer and return the length of serialization */
+ abstract public int peekLength(ByteBuffer in);
+
+ /** convert from String to obj */
+ abstract public T valueOf(byte[] value);
+
+ public T valueOf(String value) {
+ try {
+ return valueOf(value.getBytes("UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** convert from obj to string */
+ public String toString(T value) {
+ if (value == null)
+ return "NULL";
+ else
+ return value.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
index ea2678c..2874a57 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.io.DoubleWritable;
* @author yangli9
*
*/
-public class DoubleSerializer extends MeasureSerializer<DoubleWritable> {
+public class DoubleSerializer extends DataTypeSerializer<DoubleWritable> {
// avoid mass object creation
DoubleWritable current = new DoubleWritable();
@@ -44,6 +44,11 @@ public class DoubleSerializer extends MeasureSerializer<DoubleWritable> {
}
@Override
+ public int peekLength(ByteBuffer in) {
+ return 8;
+ }
+
+ @Override
public DoubleWritable valueOf(byte[] value) {
if (value == null)
current.set(0d);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
index e3fcbee..2303c82 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
* @author yangli9
*
*/
-public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
+public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
HyperLogLogPlusCounter current;
@@ -55,6 +55,11 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
}
@Override
+ public int peekLength(ByteBuffer in) {
+ return current.peekLength(in);
+ }
+
+ @Override
public HyperLogLogPlusCounter valueOf(byte[] value) {
current.clear();
if (value == null)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
index 90c1ab9..9dca987 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
@@ -22,14 +22,13 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
-
import org.apache.kylin.common.util.BytesUtil;
/**
* @author yangli9
*
*/
-public class LongSerializer extends MeasureSerializer<LongWritable> {
+public class LongSerializer extends DataTypeSerializer<LongWritable> {
// avoid mass object creation
LongWritable current = new LongWritable();
@@ -46,6 +45,17 @@ public class LongSerializer extends MeasureSerializer<LongWritable> {
}
@Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ BytesUtil.readVLong(in);
+ int len = in.position() - mark;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
public LongWritable valueOf(byte[] value) {
if (value == null)
current.set(0L);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
index 6fd928f..95a246c 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
@@ -33,7 +33,7 @@ import org.apache.kylin.metadata.model.MeasureDesc;
public class MeasureCodec {
int nMeasures;
- MeasureSerializer[] serializers;
+ DataTypeSerializer[] serializers;
public MeasureCodec(Collection<MeasureDesc> measureDescs) {
this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
@@ -53,14 +53,14 @@ public class MeasureCodec {
private void init(String[] dataTypes) {
nMeasures = dataTypes.length;
- serializers = new MeasureSerializer[nMeasures];
+ serializers = new DataTypeSerializer[nMeasures];
for (int i = 0; i < nMeasures; i++) {
- serializers[i] = MeasureSerializer.create(dataTypes[i]);
+ serializers[i] = DataTypeSerializer.create(dataTypes[i]);
}
}
- public MeasureSerializer getSerializer(int idx) {
+ public DataTypeSerializer getSerializer(int idx) {
return serializers[idx];
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureSerializer.java
deleted file mode 100644
index e245bc2..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureSerializer.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.util.HashMap;
-
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- * @author yangli9
- *
- */
-abstract public class MeasureSerializer<T> implements BytesSerializer<T> {
-
- final static HashMap<String, Class<?>> implementations = new HashMap<String, Class<?>>();
- static {
- implementations.put("decimal", BigDecimalSerializer.class);
- implementations.put("double", DoubleSerializer.class);
- implementations.put("float", DoubleSerializer.class);
- implementations.put("bigint", LongSerializer.class);
- implementations.put("long", LongSerializer.class);
- implementations.put("integer", LongSerializer.class);
- implementations.put("int", LongSerializer.class);
- }
-
- public static MeasureSerializer<?> create(String dataType) {
- DataType type = DataType.getInstance(dataType);
- if (type.isHLLC()) {
- return new HLLCSerializer(type.getPrecision());
- }
-
- Class<?> clz = implementations.get(type.getName());
- if (clz == null)
- throw new RuntimeException("No MeasureSerializer for type " + dataType);
-
- try {
- return (MeasureSerializer<?>) clz.newInstance();
- } catch (Exception e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- abstract public T valueOf(byte[] value);
-
- public String toString(T value) {
- if (value == null)
- return "NULL";
- else
- return value.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
new file mode 100644
index 0000000..9e29b52
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
@@ -0,0 +1,68 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+import java.io.Flushable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.storage.gridtable.GTStore.GTBlockWriter;
+
+public class GTBuilder implements Closeable, Flushable {
+
+ final private GTInfo info;
+ final private GTBlockWriter writer;
+
+ private GTRowBlock block;
+
+ public GTBuilder(GTInfo info, int shard, GTStore store) {
+ this.info = info;
+ this.writer = store.rebuild(shard);
+ this.block = new GTRowBlock(info);
+ }
+
+ public void write(GTRecord r) throws IOException {
+ // add record to block
+ if (block.isEmpty()) {
+ makePrimaryKey(r, block.primaryKey);
+ }
+ for (int c = 0; c < info.nColBlocks; c++) {
+ ByteBuffer cellBuf = block.cellBlocks[c];
+ for (int i = info.colBlockCuts[c], end = info.colBlockCuts[c + 1]; i < end; i++) {
+ append(cellBuf, r.cols[i]);
+ }
+ }
+
+ block.nRows++;
+ if (block.nRows >= info.rowBlockSize || info.isRowBlockEnabled() == false) {
+ flush();
+ }
+ }
+
+ private void makePrimaryKey(GTRecord r, ByteBuffer buf) {
+ buf.clear();
+
+ for (int i : info.primaryKey) {
+ append(buf, r.cols[i]);
+ }
+
+ buf.flip();
+ }
+
+ private void append(ByteBuffer buf, ByteBuffer data) {
+ buf.put(data.array(), data.arrayOffset(), data.limit());
+ }
+
+ @Override
+ public void flush() throws IOException {
+ writer.write(block);
+ block.clear();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (block.isEmpty() == false) {
+ flush();
+ }
+ writer.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
new file mode 100644
index 0000000..11a3083
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -0,0 +1,52 @@
+package org.apache.kylin.storage.gridtable;
+
+import org.apache.kylin.metadata.model.DataType;
+
+public class GTInfo {
+
+ // column schema
+ int nColumns;
+ DataType[] colType;
+ boolean[] colIsMetrics;
+ int maxRecordLength; // column length can vary
+
+ // grid info
+ int[] primaryKey; // columns sorted and unique
+ int rowBlockSize; // 0: no row block
+ int nColBlocks;
+ int[] colBlockCuts; // [i]: the start index of i.th block; [i+1]: the exclusive end index of i.th block
+
+ // sharding & rowkey
+ int nShards; // 0: no sharding
+ byte[] rowkeyPrefix;
+
+ public boolean isShardingEnabled() {
+ return nShards > 0;
+ }
+
+ public boolean isRowBlockEnabled() {
+ return rowBlockSize > 0;
+ }
+
+ void validate() {
+ if (nColBlocks != colBlockCuts.length - 1) {
+ throw new IllegalStateException();
+ }
+
+ for (int i = 0; i < nColBlocks; i++) {
+ int cbStart = colBlockCuts[i], cbEnd = colBlockCuts[i + 1];
+ if (cbStart >= cbEnd)
+ throw new IllegalStateException();
+
+ // when row block disabled, primary key is persisted on rowkey, thus shall not repeat in column block
+ if (isRowBlockEnabled() == false) {
+ for (int ii : primaryKey) {
+ if (ii >= cbStart && ii < cbEnd)
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
new file mode 100644
index 0000000..7943965
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -0,0 +1,15 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.nio.ByteBuffer;
+
+public class GTRecord {
+
+ final GTInfo info;
+ final ByteBuffer[] cols;
+
+ public GTRecord(GTInfo info) {
+ this.info = info;
+ this.cols = new ByteBuffer[info.nColumns];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
new file mode 100644
index 0000000..4526ef4
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
@@ -0,0 +1,33 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.nio.ByteBuffer;
+
+class GTRowBlock {
+
+ private static final int PRIMARY_KEY_CAPACITY = 2048;
+ private static final int CELL_BLOCK_CAPACITY = 128 * 1024;
+
+ int nRows;
+ ByteBuffer primaryKey; // the primary key of the first row
+ ByteBuffer[] cellBlocks; // cells for each column block
+
+ public GTRowBlock(GTInfo info) {
+ primaryKey = ByteBuffer.allocate(PRIMARY_KEY_CAPACITY);
+ cellBlocks = new ByteBuffer[info.nColBlocks];
+ for (int i = 0; i < cellBlocks.length; i++) {
+ cellBlocks[i] = ByteBuffer.allocate(CELL_BLOCK_CAPACITY);
+ }
+ }
+
+ public boolean isEmpty() {
+ return nRows == 0;
+ }
+
+ public void clear() {
+ nRows = 0;
+ primaryKey.clear();
+ for (int i = 0; i < cellBlocks.length; i++) {
+ cellBlocks[i].clear();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
new file mode 100644
index 0000000..3ccaa00
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
@@ -0,0 +1,24 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+public interface GTStore {
+
+ public GTInfo getInfo();
+
+ public String getStorageDescription();
+
+ public GTBlockWriter rebuild(int shard);
+
+ public GTBlockScanner scan(ByteBuffer pkStart, ByteBuffer pkEndExclusive, int[] colBlocks);
+
+ public interface GTBlockWriter extends Closeable {
+ void write(GTRowBlock block) throws IOException;
+ }
+
+ public interface GTBlockScanner extends Iterator<GTRowBlock>, Closeable {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
new file mode 100644
index 0000000..7eb8657
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
@@ -0,0 +1,19 @@
+package org.apache.kylin.storage.gridtable;
+
+public class GridTable {
+
+ final GTInfo info;
+
+ public GridTable(GTInfo info, GTStore store) {
+ this.info = info;
+ }
+
+ public GTBuilder rebuild(GTStore store) {
+ assert info.nShards == 0;
+ return rebuild(store, 0);
+ }
+
+ public GTBuilder rebuild(GTStore store, int shard) {
+ return new GTBuilder(info, shard, store);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
new file mode 100644
index 0000000..d62d478
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
@@ -0,0 +1,11 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+import org.apache.hadoop.hbase.util.Pair;
+
+public interface IKVStoreReader extends Iterator<Pair<ByteBuffer, ByteBuffer>>, Closeable {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
new file mode 100644
index 0000000..8eeba95
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
@@ -0,0 +1,11 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface IKVStoreWriter extends Closeable {
+
+ void write(ByteBuffer key, ByteBuffer value) throws IOException;
+
+}
[26/50] incubator-kylin git commit: Merge pull request #441 from
mustangore/fix-jdbc-wrap-values
Posted by li...@apache.org.
Merge pull request #441 from mustangore/fix-jdbc-wrap-values
[Kylin-626] Fix JDBC driver when returning float or double values
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b6a2bd0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b6a2bd0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b6a2bd0b
Branch: refs/heads/streaming
Commit: b6a2bd0ba1ef299bb35659725864f36250a10edf
Parents: e6b7e27 64eeed3
Author: liyang-gmt8 <li...@apache.org>
Authored: Mon Mar 9 18:09:17 2015 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Mon Mar 9 18:09:17 2015 +0800
----------------------------------------------------------------------
jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[43/50] incubator-kylin git commit: Merge branch 'staging' of
https://github.com/KylinOLAP/Kylin into staging
Posted by li...@apache.org.
Merge branch 'staging' of https://github.com/KylinOLAP/Kylin into staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/59a1d665
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/59a1d665
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/59a1d665
Branch: refs/heads/streaming
Commit: 59a1d6658301cdd6d4cbcd52daafdb1480389d3a
Parents: 76e5fd3 5f9e2c9
Author: lukehan <lu...@apache.org>
Authored: Tue Mar 10 09:47:08 2015 -0700
Committer: lukehan <lu...@apache.org>
Committed: Tue Mar 10 09:47:08 2015 -0700
----------------------------------------------------------------------
.../kylin/job/execution/AbstractExecutable.java | 10 +-
server/src/main/resources/kylinSecurity.xml | 2 +-
webapp/app/js/controllers/auth.js | 2 -
webapp/app/js/controllers/cube.js | 2 +-
webapp/app/js/controllers/cubeAdvanceSetting.js | 1 +
webapp/app/js/controllers/cubeDimensions.js | 1 -
webapp/app/js/controllers/cubeModel.js | 21 +--
webapp/app/js/filters/filter.js | 1 +
webapp/app/js/listeners.js | 7 -
webapp/app/less/component.less | 17 +--
.../cubeDesigner/advanced_settings.html | 141 ++++++++++---------
.../app/partials/cubeDesigner/data_model.html | 12 +-
.../app/partials/cubeDesigner/dimensions.html | 2 +-
webapp/app/partials/cubes/cube_schema.html | 2 +-
14 files changed, 115 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
[47/50] incubator-kylin git commit: Add a log message when a table wasn’t loaded successfully
Posted by li...@apache.org.
Add a log message when a table wasn’t loaded successfully
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/54dc31f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/54dc31f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/54dc31f7
Branch: refs/heads/streaming
Commit: 54dc31f72af63bc37deb5cd60261f880c8de2b18
Parents: 2bbaee6
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Mar 11 11:32:07 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Mar 11 11:32:07 2015 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/metadata/MetadataManager.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/54dc31f7/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 97d5d51..aedefd4 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -257,6 +257,7 @@ public class MetadataManager {
ResourceStore store = getStore();
TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER);
if (t == null) {
+ logger.error("Didn't load table at " + path);
return null;
}
t.init();
[31/50] incubator-kylin git commit: update download page
Posted by li...@apache.org.
update download page
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/30991df5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/30991df5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/30991df5
Branch: refs/heads/streaming
Commit: 30991df514eca1488aeb67c4881fe957f8508a46
Parents: e6b7e27
Author: lukehan <lu...@apache.org>
Authored: Mon Mar 9 17:25:21 2015 -0700
Committer: lukehan <lu...@apache.org>
Committed: Mon Mar 9 17:25:21 2015 -0700
----------------------------------------------------------------------
docs/website/download/index.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/30991df5/docs/website/download/index.md
----------------------------------------------------------------------
diff --git a/docs/website/download/index.md b/docs/website/download/index.md
index da40c6e..170557f 100644
--- a/docs/website/download/index.md
+++ b/docs/website/download/index.md
@@ -15,7 +15,7 @@ title: Community
<p class="big_text">Binary Package (v0.7.x)</p>
<p><a href="http://kylin.incubator.apache.org/download/kylin-0.7.1-SNAPSHOT.tgz" target="_blank">v0.7.1-SNAPSHOT</a>
- <br/>Updated Date - March 2, 2015 <br/>
+ <br/>Updated Date - March 6, 2015 <br/>
</p>
</div>