You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by wa...@apache.org on 2016/03/21 13:02:08 UTC
[1/5] kylin git commit: KYLIN-1476 Support measure dictionary in 1.x
Repository: kylin
Updated Branches:
refs/heads/1.3.x 438a6ad33 -> 821ef1e1a
KYLIN-1476 Support measure dictionary in 1.x
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d517530c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d517530c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d517530c
Branch: refs/heads/1.3.x
Commit: d517530c93dc3176a52295af297afdfefdfc4b62
Parents: 438a6ad
Author: wangxiaoyu <ro...@gmail.com>
Authored: Tue Mar 8 23:39:51 2016 +0800
Committer: Xiaoyu Wang <wa...@apache.org>
Committed: Mon Mar 21 20:00:57 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 26 ++++++++++++-
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 14 +++----
.../kylin/job/cube/MergeDictionaryStep.java | 17 ++++-----
.../hadoop/cube/FactDistinctColumnsMapper.java | 40 +++++---------------
.../hadoop/cube/FactDistinctColumnsReducer.java | 12 ++----
.../hadoop/hive/CubeJoinedFlatTableDesc.java | 18 +++++++--
6 files changed, 65 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d517530c/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 88ce4f1..4615762 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -194,11 +194,11 @@ public class CubeManager implements IRealizationProvider {
public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, String factColumnsPath) throws IOException {
CubeDesc cubeDesc = cubeSeg.getCubeDesc();
- if (!cubeDesc.getRowkey().isUseDictionary(col))
+ if (!cubeDesc.getAllColumnsNeedDictionary().contains(col))
return null;
DictionaryManager dictMgr = getDictionaryManager();
- DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath);
+ DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), "true", col, factColumnsPath);
if (dictInfo != null) {
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
@@ -791,4 +791,26 @@ public class CubeManager implements IRealizationProvider {
return getCube(name);
}
+
+ /**
+ * Get the columns which need build the dictionary from fact table. (the column exists on fact and is not fk)
+ * @param cubeDesc
+ * @return
+ * @throws IOException
+ */
+ public List<TblColRef> getAllDictColumnsOnFact(CubeDesc cubeDesc) throws IOException {
+ List<TblColRef> dictionaryColumns = cubeDesc.getAllColumnsNeedDictionary();
+
+ List<TblColRef> factDictCols = new ArrayList<TblColRef>();
+ DictionaryManager dictMgr = DictionaryManager.getInstance(config);
+ for (int i = 0; i < dictionaryColumns.size(); i++) {
+ TblColRef col = dictionaryColumns.get(i);
+
+ String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), "true", col, null)[0];
+ if (cubeDesc.getModel().isFactTable(scanTable)) {
+ factDictCols.add(col);
+ }
+ }
+ return factDictCols;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d517530c/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 3cdaa93..bcbd496 100644
--- a/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -44,15 +44,13 @@ public class DictionaryGeneratorCLI {
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(config);
- for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
- logger.info("Building dictionary for " + col);
- cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath);
- }
- }
+ // dictionary
+ for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionary()) {
+ logger.info("Building dictionary for " + col);
+ cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath);
+ }
+ for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
// build snapshot
if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
// CubeSegment seg = cube.getTheOnlySegment();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d517530c/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
index 8f77558..47151a7 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
@@ -107,16 +107,13 @@ public class MergeDictionaryStep extends AbstractExecutable {
DictionaryManager dictMgr = DictionaryManager.getInstance(conf);
CubeDesc cubeDesc = cube.getDescriptor();
- for (DimensionDesc dim : cubeDesc.getDimensions()) {
- for (TblColRef col : dim.getColumnRefs()) {
- if (newSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
- String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
- if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
- colsNeedMeringDict.add(col);
- } else {
- colsNeedCopyDict.add(col);
- }
- }
+
+ for (TblColRef col : cubeDesc.getAllColumnsNeedDictionary()) {
+ String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), "true", col, null)[0];
+ if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
+ colsNeedMeringDict.add(col);
+ } else {
+ colsNeedCopyDict.add(col);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d517530c/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
index 72802aa..b70113d 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.hadoop.cube;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -34,10 +33,7 @@ import org.apache.kylin.common.mr.KylinMapper;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.job.hadoop.hive.CubeJoinedFlatTableDesc;
@@ -51,7 +47,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRec
private String cubeName;
private CubeInstance cube;
private CubeDesc cubeDesc;
- private int[] factDictCols;
+ List<TblColRef> factDictCols;
private CubeJoinedFlatTableDesc intermediateTableDesc;
@@ -60,6 +56,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRec
private int errorRecordCounter;
private HCatSchema schema = null;
+ protected int[] dictionaryColumnIndex;
@Override
protected void setup(Context context) throws IOException {
@@ -73,40 +70,23 @@ public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRec
cubeDesc = cube.getDescriptor();
intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- List<TblColRef> columns = baseCuboid.getColumns();
-
- ArrayList<Integer> factDictCols = new ArrayList<Integer>();
- RowKeyDesc rowkey = cubeDesc.getRowkey();
- DictionaryManager dictMgr = DictionaryManager.getInstance(config);
- for (int i = 0; i < columns.size(); i++) {
- TblColRef col = columns.get(i);
- if (rowkey.isUseDictionary(col) == false)
- continue;
-
- String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
- if (cubeDesc.getModel().isFactTable(scanTable)) {
- factDictCols.add(i);
- }
+ factDictCols = CubeManager.getInstance(config).getAllDictColumnsOnFact(cubeDesc);
+ dictionaryColumnIndex = new int[factDictCols.size()];
+ for (int i = 0; i < factDictCols.size(); i++) {
+ TblColRef colRef = factDictCols.get(i);
+ int columnIndexOnFlatTbl = intermediateTableDesc.getColumnIndex(colRef);
+ dictionaryColumnIndex[i] = columnIndexOnFlatTbl;
}
- this.factDictCols = new int[factDictCols.size()];
- for (int i = 0; i < factDictCols.size(); i++)
- this.factDictCols[i] = factDictCols.get(i);
-
schema = HCatInputFormat.getTableSchema(context.getConfiguration());
}
@Override
public void map(KEYIN key, HCatRecord record, Context context) throws IOException, InterruptedException {
-
try {
-
- int[] flatTableIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
HCatFieldSchema fieldSchema = null;
- for (int i : factDictCols) {
+ for (int i = 0; i < factDictCols.size(); i++) {
outputKey.set((short) i);
- fieldSchema = schema.get(flatTableIndexes[i]);
+ fieldSchema = schema.get(dictionaryColumnIndex[i]);
Object fieldValue = record.get(fieldSchema.getName(), schema);
if (fieldValue == null)
continue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d517530c/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index 89f90ba..809ced2 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -31,14 +31,12 @@ import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.metadata.model.TblColRef;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -47,8 +45,7 @@ import java.util.List;
*/
public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text, NullWritable, Text> {
- private List<TblColRef> columnList = new ArrayList<TblColRef>();
-
+ private List<TblColRef> factDictCols;
@Override
protected void setup(Context context) throws IOException {
super.publishConfiguration(context.getConfiguration());
@@ -58,15 +55,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text
String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
CubeDesc cubeDesc = cube.getDescriptor();
-
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- columnList = baseCuboid.getColumns();
+ factDictCols = CubeManager.getInstance(config).getAllDictColumnsOnFact(cubeDesc);
}
@Override
public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
- TblColRef col = columnList.get(key.get());
+ TblColRef col = factDictCols.get(key.get());
HashSet<ByteArray> set = new HashSet<ByteArray>();
for (Text textValue : values) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d517530c/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
index b2bed9f..72dd400 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
@@ -48,9 +48,12 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
private List<IntermediateColumnDesc> columnList = Lists.newArrayList();
+ private Map<String, Integer> columnIndexMap;
+
public CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment) {
this.cubeDesc = cubeDesc;
this.cubeSegment = cubeSegment;
+ this.columnIndexMap = Maps.newHashMap();
parseCubeDesc();
}
@@ -73,10 +76,9 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
this.tableName = "kylin_intermediate_" + cubeDesc.getName() + "_" + cubeSegment.getName();
}
- Map<String, Integer> dimensionIndexMap = Maps.newHashMap();
int columnIndex = 0;
for (TblColRef col : cubeDesc.listDimensionColumnsExcludingDerived()) {
- dimensionIndexMap.put(colName(col.getCanonicalName()), columnIndex);
+ columnIndexMap.put(colName(col.getCanonicalName()), columnIndex);
columnList.add(new IntermediateColumnDesc(String.valueOf(columnIndex), col));
columnIndex++;
}
@@ -86,7 +88,7 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
rowKeyColumnIndexes = new int[rowkeyColCount];
for (int i = 0; i < rowkeyColCount; i++) {
String colName = colName(cuboidColumns.get(i).getCanonicalName());
- Integer dimIdx = dimensionIndexMap.get(colName);
+ Integer dimIdx = columnIndexMap.get(colName);
if (dimIdx == null) {
throw new RuntimeException("Can't find column " + colName);
}
@@ -108,6 +110,7 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
measureColumnIndexes[i][j] = contains(columnList, c);
if (measureColumnIndexes[i][j] < 0) {
measureColumnIndexes[i][j] = columnIndex;
+ columnIndexMap.put(colName(c.getCanonicalName()), columnIndex);
columnList.add(new IntermediateColumnDesc(String.valueOf(columnIndex), c));
columnIndex++;
}
@@ -172,4 +175,13 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
private static String colName(String canonicalColName) {
return canonicalColName.replace(".", "_");
}
+
+ public int getColumnIndex(TblColRef colRef) {
+ String key = colName(colRef.getCanonicalName());
+ Integer index = columnIndexMap.get(key);
+ if (index == null)
+ throw new IllegalArgumentException("Column " + colRef.toString() + " wasn't found on flat table.");
+
+ return index.intValue();
+ }
}
[3/5] kylin git commit: update hbase-export.tar.gz after add raw
measures in cube desc
Posted by wa...@apache.org.
update hbase-export.tar.gz after add raw measures in cube desc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/97b2786a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/97b2786a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/97b2786a
Branch: refs/heads/1.3.x
Commit: 97b2786a6763d293ba3c7f133b15aba38fced5ee
Parents: a48d100
Author: wangxiaoyu <ro...@gmail.com>
Authored: Wed Mar 9 22:02:35 2016 +0800
Committer: Xiaoyu Wang <wa...@apache.org>
Committed: Mon Mar 21 20:01:04 2016 +0800
----------------------------------------------------------------------
.../minicluster/hbase-export.tar.gz | Bin 29527690 -> 29746922 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/97b2786a/examples/test_case_data/minicluster/hbase-export.tar.gz
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/hbase-export.tar.gz b/examples/test_case_data/minicluster/hbase-export.tar.gz
index fa06943..f9bd01b 100755
Binary files a/examples/test_case_data/minicluster/hbase-export.tar.gz and b/examples/test_case_data/minicluster/hbase-export.tar.gz differ
[2/5] kylin git commit: KYLIN-1122 Kylin support detail data query
from fact table
Posted by wa...@apache.org.
KYLIN-1122 Kylin support detail data query from fact table
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a48d1004
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a48d1004
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a48d1004
Branch: refs/heads/1.3.x
Commit: a48d1004119bd95c65482cc8419e3e8dee628b7e
Parents: d517530
Author: wangxiaoyu <ro...@gmail.com>
Authored: Tue Mar 8 23:42:14 2016 +0800
Committer: Xiaoyu Wang <wa...@apache.org>
Committed: Mon Mar 21 20:00:58 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ByteArray.java | 106 ++++++-
.../kylin/cube/CubeCapabilityChecker.java | 6 +-
.../test_kylin_cube_without_slr_desc.json | 60 +++-
...t_kylin_cube_without_slr_left_join_desc.json | 58 +++-
.../kylin/measure/MeasureTypeFactory.java | 2 +
.../apache/kylin/measure/raw/RawAggregator.java | 65 +++++
.../kylin/measure/raw/RawMeasureType.java | 280 +++++++++++++++++++
.../apache/kylin/measure/raw/RawSerializer.java | 110 ++++++++
.../metadata/datatype/DataTypeSerializer.java | 1 +
.../kylin/metadata/realization/SQLDigest.java | 1 +
.../org/apache/kylin/metadata/tuple/Tuple.java | 9 +-
.../kylin/measure/raw/RawAggregatorTest.java | 52 ++++
.../kylin/measure/raw/RawSerializerTest.java | 126 +++++++++
.../kylin/query/enumerator/CubeEnumerator.java | 1 +
.../apache/kylin/query/test/KylinQueryTest.java | 7 +-
.../apache/kylin/query/test/KylinTestBase.java | 2 +-
.../test/resources/query/sql_raw/query01.sql | 19 ++
.../test/resources/query/sql_raw/query02.sql | 19 ++
.../test/resources/query/sql_raw/query03.sql | 19 ++
.../test/resources/query/sql_raw/query04.sql | 19 ++
.../test/resources/query/sql_raw/query05.sql | 19 ++
.../test/resources/query/sql_raw/query06.sql | 19 ++
webapp/app/js/controllers/cubeSchema.js | 3 +
webapp/app/js/model/cubeConfig.js | 2 +-
24 files changed, 985 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index 92e0da2..afbe968 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -14,26 +14,97 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- */
+*/
package org.apache.kylin.common.util;
-import java.util.Arrays;
+import java.io.Serializable;
/**
* @author yangli9
*/
-public class ByteArray implements Comparable<ByteArray> {
+public class ByteArray implements Comparable<ByteArray>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public static ByteArray allocate(int length) {
+ return new ByteArray(new byte[length]);
+ }
public byte[] data;
+ private int offset;
+ private int length;
+
+ public ByteArray() {
+ this(null, 0, 0);
+ }
+
+ public ByteArray(int capacity) {
+ this(new byte[capacity], 0, capacity);
+ }
public ByteArray(byte[] data) {
+ this(data, 0, data == null ? 0 : data.length);
+ }
+
+ public ByteArray(byte[] data, int offset, int length) {
this.data = data;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ public byte[] array() {
+ return data;
+ }
+
+ public int offset() {
+ return offset;
+ }
+
+ public int length() {
+ return length;
+ }
+
+ public void set(byte[] array) {
+ set(array, 0, array.length);
+ }
+
+ public void set(byte[] array, int offset, int length) {
+ this.data = array;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ public void set(ByteArray o) {
+ set(o.data, o.offset, o.length);
+ }
+
+ public void set(int offset, int length) {
+ this.offset = offset;
+ this.length = length;
+ }
+
+ public void setLength(int length) {
+ this.length = length;
+ }
+
+ public ByteArray copy() {
+ ByteArray copy = new ByteArray(length);
+ copy.copyFrom(this);
+ return copy;
+ }
+
+ public void copyFrom(ByteArray other) {
+ System.arraycopy(other.array(), other.offset, data, offset, other.length);
+ this.length = other.length;
}
@Override
public int hashCode() {
- return Bytes.hashCode(data);
+ if (data == null)
+ return 0;
+ else
+ return Bytes.hashCode(data, offset, length);
}
@Override
@@ -44,14 +115,33 @@ public class ByteArray implements Comparable<ByteArray> {
return false;
if (getClass() != obj.getClass())
return false;
- ByteArray other = (ByteArray) obj;
- if (!Arrays.equals(data, other.data))
+ ByteArray o = (ByteArray) obj;
+ if (this.data == null && o.data == null)
+ return true;
+ else if (this.data == null || o.data == null)
return false;
- return true;
+ else
+ return Bytes.equals(this.data, this.offset, this.length, o.data, o.offset, o.length);
}
@Override
public int compareTo(ByteArray o) {
- return Bytes.compareTo(this.data, o.data);
+ if (this.data == null && o.data == null)
+ return 0;
+ else if (this.data == null)
+ return -1;
+ else if (o.data == null)
+ return 1;
+ else
+ return Bytes.compareTo(this.data, this.offset, this.length, o.data, o.offset, o.length);
}
+
+ @Override
+ public String toString() {
+ if (data == null)
+ return null;
+ else
+ return Bytes.toStringBinary(data, offset, length);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 5668030..fce11c7 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -64,9 +64,7 @@ public class CubeCapabilityChecker {
Collection<FunctionDesc> unmatchedAggregations = unmatchedAggregations(aggrFunctions, cube);
// try custom measure types
- if (!unmatchedDimensions.isEmpty() || !unmatchedAggregations.isEmpty()) {
- tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, digest, cube, result);
- }
+ tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, digest, cube, result);
// try dimension-as-measure
if (!unmatchedAggregations.isEmpty()) {
@@ -180,8 +178,6 @@ public class CubeCapabilityChecker {
private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) {
CubeDesc cubeDesc = cube.getDescriptor();
for (MeasureDesc measure : cubeDesc.getMeasures()) {
- if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
- break;
MeasureType<?> measureType = measure.getFunction().getMeasureType();
if (measureType instanceof BasicMeasureType)
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index f31e358..2dcd963 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -179,6 +179,58 @@
"returntype": "bitmap"
},
"dependent_measure_ref": null
+ },
+ {
+ "id": 9,
+ "name": "CAL_DT_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "CAL_DT"
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 10,
+ "name": "LSTG_FORMAT_NAME_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "LSTG_FORMAT_NAME"
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 11,
+ "name": "LEAF_CATEG_ID_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "LEAF_CATEG_ID"
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 12,
+ "name": "PRICE_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE"
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
}
],
"rowkey": {
@@ -263,7 +315,11 @@
"gmv_min",
"gmv_max",
"trans_cnt",
- "item_count_sum"
+ "item_count_sum",
+ "CAL_DT_RAW",
+ "LSTG_FORMAT_NAME_RAW",
+ "LEAF_CATEG_ID_RAW",
+ "PRICE_RAW"
]
}
]
@@ -284,4 +340,4 @@
]
},
"notify_list": null
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index ef27b03..a2632ea 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -179,6 +179,58 @@
"returntype": "bitmap"
},
"dependent_measure_ref": null
+ },
+ {
+ "id": 9,
+ "name": "CAL_DT_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "CAL_DT"
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 10,
+ "name": "LSTG_FORMAT_NAME_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "LSTG_FORMAT_NAME"
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 11,
+ "name": "LEAF_CATEG_ID_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "LEAF_CATEG_ID"
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
+ },
+ {
+ "id": 12,
+ "name": "PRICE_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE"
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
}
],
"rowkey": {
@@ -263,7 +315,11 @@
"gmv_min",
"gmv_max",
"trans_cnt",
- "item_count_sum"
+ "item_count_sum",
+ "CAL_DT_RAW",
+ "LSTG_FORMAT_NAME_RAW",
+ "LEAF_CATEG_ID_RAW",
+ "PRICE_RAW"
]
}
]
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index 33d94fb..1a77f72 100644
--- a/metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ b/metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.kylin.measure.basic.BasicMeasureType;
import org.apache.kylin.measure.bitmap.BitmapMeasureType;
import org.apache.kylin.measure.hllc.HLLCMeasureType;
+import org.apache.kylin.measure.raw.RawMeasureType;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
@@ -48,6 +49,7 @@ abstract public class MeasureTypeFactory<T> {
// two built-in advanced measure types
factoryInsts.add(new HLLCMeasureType.Factory());
factoryInsts.add(new BitmapMeasureType.Factory());
+ factoryInsts.add(new RawMeasureType.Factory());
/*
* Maybe do classpath search for more custom measure types?
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java b/metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
new file mode 100644
index 0000000..c2fe30d
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.raw;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.measure.MeasureAggregator;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * RAW data Aggregator
+ */
+public class RawAggregator extends MeasureAggregator<List<ByteArray>> {
+
+ List<ByteArray> list = null;
+
+ @Override
+ public void reset() {
+ list = null;
+ }
+
+ @Override
+ public void aggregate(List<ByteArray> value) {
+ if(value != null) {
+ if (list == null) {
+ list = new ArrayList<>(value.size());
+ }
+ list.addAll(value);
+ }
+ }
+
+ @Override
+ public List<ByteArray> getState() {
+ return list;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ int bytes = 0;
+ if(list != null) {
+ for (ByteArray array : list) {
+ bytes += array.length() + 1;
+ }
+ }
+ return bytes;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
new file mode 100644
index 0000000..c6b1637
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.raw;
+
+import org.apache.kylin.common.util.*;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.CapabilityResult;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class RawMeasureType extends MeasureType<List<ByteArray>> {
+
+ private static final Logger logger = LoggerFactory.getLogger(RawMeasureType.class);
+
+ public static final String FUNC_RAW = "RAW";
+ public static final String DATATYPE_RAW = "raw";
+
+ public static class Factory extends MeasureTypeFactory<List<ByteArray>> {
+
+ @Override
+ public MeasureType<List<ByteArray>> createMeasureType(String funcName, DataType dataType) {
+ return new RawMeasureType(funcName, dataType);
+ }
+
+ @Override
+ public String getAggrFunctionName() {
+ return FUNC_RAW;
+ }
+
+ @Override
+ public String getAggrDataTypeName() {
+ return DATATYPE_RAW;
+ }
+
+ @Override
+ public Class<? extends DataTypeSerializer<List<ByteArray>>> getAggrDataTypeSerializer() {
+ return RawSerializer.class;
+ }
+ }
+
+ private final DataType dataType;
+
+ public RawMeasureType(String funcName, DataType dataType) {
+ this.dataType = dataType;
+ }
+
+ public void validate(FunctionDesc functionDesc) throws IllegalArgumentException {
+ validate(functionDesc.getExpression(), functionDesc.getReturnDataType(), true);
+ }
+
+ private void validate(String funcName, DataType dataType, boolean checkDataType) {
+ if (FUNC_RAW.equals(funcName) == false)
+ throw new IllegalArgumentException();
+
+ if (DATATYPE_RAW.equals(dataType.getName()) == false)
+ throw new IllegalArgumentException();
+
+ }
+
+ @Override
+ public boolean isMemoryHungry() {
+ return true;
+ }
+
+ @Override
+ public MeasureIngester<List<ByteArray>> newIngester() {
+ return new MeasureIngester<List<ByteArray>>() {
+ //encode measure value to dictionary
+ @Override
+ public List<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ if (values.length != 1)
+ throw new IllegalArgumentException();
+
+ //source input column value
+ String literal = values[0];
+ // encode literal using dictionary
+ TblColRef literalCol = getRawColumn(measureDesc.getFunction());
+ Dictionary<String> dictionary = dictionaryMap.get(literalCol);
+ int keyEncodedValue = dictionary.getIdFromValue(literal);
+
+ ByteArray key = new ByteArray(dictionary.getSizeOfId());
+ BytesUtil.writeUnsigned(keyEncodedValue, key.array(), key.offset(), dictionary.getSizeOfId());
+
+ List<ByteArray> valueList = new ArrayList<ByteArray>(1);
+ valueList.add(key);
+ return valueList;
+ }
+
+ //merge measure dictionary
+ @Override
+ public List<ByteArray> reEncodeDictionary(List<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
+ TblColRef colRef = getRawColumn(measureDesc.getFunction());
+ Dictionary<String> sourceDict = oldDicts.get(colRef);
+ Dictionary<String> mergedDict = newDicts.get(colRef);
+
+ int valueSize = value.size();
+ byte[] newIdBuf = new byte[valueSize * mergedDict.getSizeOfId()];
+ byte[] literal = new byte[sourceDict.getSizeOfValue()];
+
+ int bufOffset = 0;
+ for (ByteArray c : value) {
+ int oldId = BytesUtil.readUnsigned(c.array(), c.offset(), c.length());
+ int newId;
+ int size = sourceDict.getValueBytesFromId(oldId, literal, 0);
+ if (size < 0) {
+ newId = mergedDict.nullId();
+ } else {
+ newId = mergedDict.getIdFromValueBytes(literal, 0, size);
+ }
+ BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId());
+ c.set(newIdBuf, bufOffset, mergedDict.getSizeOfId());
+ bufOffset += mergedDict.getSizeOfId();
+ }
+ return value;
+ }
+ };
+ }
+
+ @Override
+ public MeasureAggregator<List<ByteArray>> newAggregator() {
+ return new RawAggregator();
+ }
+
+ @Override
+ public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) {
+ TblColRef literalCol = functionDesc.getParameter().getColRefs().get(0);
+ return Collections.singletonList(literalCol);
+ }
+
+ public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, MeasureDesc measureDesc) {
+ //is raw query
+ if (digest.aggregations.size() != 0 || digest.metricColumns.size() != 0)
+ return null;
+
+ TblColRef rawColumn = getRawColumn(measureDesc.getFunction());
+ if (!digest.allColumns.isEmpty() && !digest.allColumns.contains(rawColumn)) {
+ return null;
+ }
+
+ unmatchedAggregations.remove(measureDesc.getFunction());
+
+ //contain one raw measure : cost * 0.9
+ return new CapabilityResult.CapabilityInfluence(){
+ @Override
+ public double suggestCostMultiplier() {
+ return 0.9;
+ }
+ };
+ }
+
+ @Override
+ public boolean needRewrite() {
+ return false;
+ }
+
+ @Override
+ public Class<?> getRewriteCalciteAggrFunctionClass() {
+ return null;
+ }
+
+ @Override
+ public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
+ if (sqlDigest.isRawQuery) {
+ TblColRef col = this.getRawColumn(measureDesc.getFunction());
+ ParameterDesc colParameter = new ParameterDesc();
+ colParameter.setType("column");
+ colParameter.setValue(col.getName());
+ FunctionDesc rawFunc = new FunctionDesc();
+ rawFunc.setExpression("RAW");
+ rawFunc.setParameter(colParameter);
+
+ if (sqlDigest.allColumns.contains(col)) {
+ if (measureDesc.getFunction().equals(rawFunc)) {
+ FunctionDesc sumFunc = new FunctionDesc();
+ sumFunc.setExpression("SUM");
+ sumFunc.setParameter(colParameter);
+ sqlDigest.aggregations.remove(sumFunc);
+ sqlDigest.aggregations.add(rawFunc);
+ logger.info("Add RAW measure on column " + col);
+ }
+ if (!sqlDigest.metricColumns.contains(col)) {
+ sqlDigest.metricColumns.add(col);
+ }
+ }
+ }
+ }
+
+ @Override
+ public boolean needAdvancedTupleFilling() {
+ return true;
+ }
+
+ @Override
+ public void fillTupleSimply(Tuple tuple, int indexInTuple, Object measureValue) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo tupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ final TblColRef literalCol = getRawColumn(function);
+ final Dictionary<String> rawColDict = dictionaryMap.get(literalCol);
+ final String rewriteFieldName = function.getRewriteFieldName();
+
+ return new IAdvMeasureFiller() {
+ private List<ByteArray> rawList;
+ private Iterator<ByteArray> rawIterator;
+ private int expectRow = 0;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void reload(Object measureValue) {
+ this.rawList = (List<ByteArray>) measureValue;
+ this.rawIterator = rawList.iterator();
+ this.expectRow = 0;
+ }
+
+ @Override
+ public int getNumOfRows() {
+ return rawList.size();
+ }
+
+ @Override
+ public void fillTuplle(Tuple tuple, int row) {
+ if (expectRow++ != row)
+ throw new IllegalStateException();
+
+ ByteArray raw = rawIterator.next();
+ int key = BytesUtil.readUnsigned(raw.array(), raw.offset(), raw.length());
+ String colValue = rawColDict.getValueFromId(key);
+ if (!tuple.getAllFields().contains(literalCol.getName())) {
+ if (tuple.getAllFields().contains(rewriteFieldName)) {
+ int literalTupleIdx = tuple.getInfo().getFieldIndex(rewriteFieldName);
+ tuple.getInfo().setField(literalCol.getName(), literalCol, literalCol.getType().getName(), literalTupleIdx);
+ }
+ }
+ tuple.setDimensionValue(literalCol.getName(), colValue);
+ }
+ };
+ }
+
+ private TblColRef getRawColumn(FunctionDesc functionDesc) {
+ return functionDesc.getParameter().getColRefs().get(0);
+ }
+
+ @Override
+ public boolean onlyAggrInBaseCuboid() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java b/metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
new file mode 100644
index 0000000..3d7e0b6
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.raw;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+@SuppressWarnings("unused")
+public class RawSerializer extends DataTypeSerializer<List<ByteArray>> {
+
+ //one dictionary id value need 1~4 bytes,length need 1~4 bytes, this buffer can contain 1024/(2 to 8) * 1024 values
+ //FIXME to config this and RowConstants.ROWVALUE_BUFFER_SIZE in properties file
+ public static final int RAW_BUFFER_SIZE = 1024 * 1024;//1M
+
+ private ThreadLocal<List<ByteArray>> current = new ThreadLocal<>();
+
+ public RawSerializer(DataType dataType) {
+ }
+
+ private List<ByteArray> current() {
+ List<ByteArray> l = current.get();
+ if (l == null) {
+ l = new ArrayList<ByteArray>();
+ current.set(l);
+ }
+ return l;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ int len = 0;
+ if (in.hasRemaining()) {
+ int size = BytesUtil.readVInt(in);
+ len = in.position() - mark;
+ for (int i = 0; i < size; i++) {
+ int length = BytesUtil.peekByteArrayLength(in);
+ in.position(in.position() + length);
+ len += length;
+ }
+ }
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public int maxLength() {
+ return RAW_BUFFER_SIZE;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+ @Override
+ public void serialize(List<ByteArray> values, ByteBuffer out) {
+ if (values == null) {
+ BytesUtil.writeVInt(0, out);
+ } else {
+ BytesUtil.writeVInt(values.size(), out);
+ for (ByteArray array : values) {
+ if (!out.hasRemaining() || out.remaining() < array.length()) {
+ throw new RuntimeException("BufferOverflow! Please use one higher cardinality column for dimension column when build RAW cube!");
+ }
+ BytesUtil.writeByteArray(BytesUtil.subarray(array.array(), array.offset(), array.offset() + array.length()), out);
+ }
+ }
+ }
+
+ @Override
+ public List<ByteArray> deserialize(ByteBuffer in) {
+ List<ByteArray> values = current();
+ values.clear();
+ int size = BytesUtil.readVInt(in);
+ if (size >= 0) {
+ for (int i = 0; i < size; i++) {
+ ByteArray ba = new ByteArray(BytesUtil.readByteArray(in));
+ if (ba.length() != 0){
+ values.add(ba);
+ }
+ }
+ } else {
+ throw new RuntimeException("Read error data size:" + size);
+ }
+ return values;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
index af4f97f..56e3778 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
@@ -32,6 +32,7 @@ abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
final static Map<String, Class<?>> implementations = Maps.newHashMap();
static {
+ implementations.put("char", StringSerializer.class);
implementations.put("varchar", StringSerializer.class);
implementations.put("decimal", BigDecimalSerializer.class);
implementations.put("double", DoubleSerializer.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index a3d9763..3e124ba 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -37,6 +37,7 @@ public class SQLDigest {
public Collection<TblColRef> filterColumns;
public Collection<TblColRef> metricColumns;
public Collection<FunctionDesc> aggregations;
+ public boolean isRawQuery = false;
public SQLDigest(String factTable, TupleFilter filter, Collection<JoinDesc> joinDescs, Collection<TblColRef> allColumns, //
Collection<TblColRef> groupbyColumns, Collection<TblColRef> filterColumns, Collection<TblColRef> aggregatedColumns, Collection<FunctionDesc> aggregateFunnc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index 321a768..9c2416e 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -96,10 +96,17 @@ public class Tuple implements ITuple {
// BigDecimal during cube build for best precision
if ("double".equals(dataType) && fieldValue instanceof BigDecimal) {
fieldValue = ((BigDecimal) fieldValue).doubleValue();
- } else if ("integer".equals(dataType) && !(fieldValue instanceof Integer)) {
+ } else if ("integer".equals(dataType) && fieldValue instanceof Number) {
fieldValue = ((Number) fieldValue).intValue();
} else if ("float".equals(dataType) && fieldValue instanceof BigDecimal) {
fieldValue = ((BigDecimal) fieldValue).floatValue();
+ } else if ("date".equals(dataType) && fieldValue instanceof Long) {
+ long millis = ((Long)fieldValue).longValue();
+ fieldValue = (int) (millis / (1000 * 3600 * 24));
+ } else if ("smallint".equals(dataType) && fieldValue instanceof Long) {
+ fieldValue = ((Long)fieldValue).shortValue();
+ } else if ((!"varchar".equals(dataType) || !"char".equals(dataType)) && fieldValue instanceof String) {
+ fieldValue = convertOptiqCellValue((String)fieldValue, dataType);
}
setFieldObjectValue(idx, fieldValue);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java b/metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java
new file mode 100644
index 0000000..ec59d36
--- /dev/null
+++ b/metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.measure.raw;
+
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class RawAggregatorTest {
+ private RawAggregator agg = new RawAggregator();
+
+ @Test
+ public void testNormal(){
+ int size = 100;
+ List<ByteArray> valueList = new ArrayList<ByteArray>(size);
+ for (Integer i = 0; i < size; i++) {
+ ByteArray key = new ByteArray(1);
+ BytesUtil.writeUnsigned(i, key.array(), 0, key.length());
+ valueList.add(key);
+ }
+ agg.aggregate(valueList);
+ agg.aggregate(valueList);
+ assertEquals(valueList.size() * 2, agg.getState().size());
+ }
+
+ @Test
+ public void testNull(){
+ agg.aggregate(null);
+ assertEquals(agg.getState(), null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java b/metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java
new file mode 100644
index 0000000..8bd09ff
--- /dev/null
+++ b/metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.measure.raw;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class RawSerializerTest {
+ static {
+ DataType.register("raw");
+ }
+
+ private RawSerializer rawSerializer = new RawSerializer(DataType.getInstance("raw"));
+
+ @Test
+ public void testPeekLength() {
+ ByteBuffer out = ByteBuffer.allocate(1024 * 1024 * 128);
+ int size = 127;
+ List<ByteArray> input = getValueList(size);
+ rawSerializer.serialize(input, out);
+ out.rewind();
+ assertEquals(size * 2 + 1, rawSerializer.peekLength(out));
+
+ size = 128;
+ out.clear();
+ input = getValueList(size);
+ rawSerializer.serialize(input, out);
+ out.rewind();
+ assertEquals(size * 2 + 2, rawSerializer.peekLength(out));
+
+ size = 255;
+ out.clear();
+ input = getValueList(size);
+ rawSerializer.serialize(input, out);
+ out.rewind();
+ assertEquals(size * 2 + 2, rawSerializer.peekLength(out));
+
+ size = 256;
+ out.clear();
+ input = getValueList(size);
+ rawSerializer.serialize(input, out);
+ out.rewind();
+ assertEquals(size * 2 + 3, rawSerializer.peekLength(out));
+
+ size = 1024 * 63;
+ out.clear();
+ input = getValueList(size);
+ rawSerializer.serialize(input, out);
+ out.rewind();
+ assertEquals(size * 2 + 3, rawSerializer.peekLength(out));
+
+ size = 1024 * 64;
+ out.clear();
+ input = getValueList(size);
+ rawSerializer.serialize(input, out);
+ out.rewind();
+ assertEquals(size * 2 + 4, rawSerializer.peekLength(out));
+ }
+
+ @Test
+ public void testNormal() {
+ List<ByteArray> input = getValueList(1024);
+ List<ByteArray> output = doSAndD(input);
+ assertEquals(input, output);
+ }
+
+ @Test
+ public void testNull() {
+ List<ByteArray> output = doSAndD(null);
+ assertEquals(output.size(), 0);
+ List<ByteArray> input = new ArrayList<ByteArray>();
+ output = doSAndD(input);
+ assertEquals(input, output);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testOverflow() {
+ List<ByteArray> input = getValueList(512 * 1024);
+ doSAndD(input);
+ }
+
+ private List<ByteArray> doSAndD(List<ByteArray> input) {
+ ByteBuffer out = ByteBuffer.allocate(rawSerializer.maxLength());
+ out.mark();
+ rawSerializer.serialize(input, out);
+ out.reset();
+ return rawSerializer.deserialize(out);
+ }
+
+ private List<ByteArray> getValueList(int size) {
+ if (size == -1) {
+ return null;
+ }
+ List<ByteArray> valueList = new ArrayList<ByteArray>(size);
+ for (Integer i = 0; i < size; i++) {
+ ByteArray key = new ByteArray(1);
+ BytesUtil.writeUnsigned(i, key.array(), 0, key.length());
+ valueList.add(key);
+ }
+ return valueList;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
index 66a4035..a556847 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
@@ -192,6 +192,7 @@ public class CubeEnumerator implements Enumerator<Object[]> {
if (!sqlDigest.groupbyColumns.isEmpty() || !sqlDigest.metricColumns.isEmpty())
return;
+ sqlDigest.isRawQuery = true;
// If no group by and metric found, then it's simple query like select ... from ... where ...,
// But we have no raw data stored, in order to return better results, we hack to output sum of metric column
logger.info("No group by and aggregation found in this query, will hack some result for better look of output...");
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index 2c6b155..3359798 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -80,7 +80,7 @@ public class KylinQueryTest extends KylinTestBase {
// Cube Connection
File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, config);
Properties props = new Properties();
- props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "10000");
+ props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "10001");
cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props);
}
@@ -264,6 +264,11 @@ public class KylinQueryTest extends KylinTestBase {
assertLimitWasEnabled();
}
+ @Test
+ public void testRawQuery() throws Exception {
+ this.execAndCompQuery("src/test/resources/query/sql_raw", null, true);
+ }
+
private void assertLimitWasEnabled() {
OLAPContext context = getFirstOLAPContext();
assertTrue(context.storageContext.isLimitEnabled());
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java b/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
index f9205b7..b3f46ca 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
@@ -336,7 +336,7 @@ public class KylinTestBase {
ITable kylinTable = executeQuery(kylinConn, queryName, sql, false);
// compare the result
- Assert.assertEquals(expectRowCount, kylinTable.getRowCount());
+ Assert.assertEquals(queryName, expectRowCount, kylinTable.getRowCount());
// Assertion.assertEquals(expectRowCount, kylinTable.getRowCount());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/query/src/test/resources/query/sql_raw/query01.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_raw/query01.sql b/query/src/test/resources/query/sql_raw/query01.sql
new file mode 100644
index 0000000..54816ea
--- /dev/null
+++ b/query/src/test/resources/query/sql_raw/query01.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select PRICE from test_kylin_fact
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/query/src/test/resources/query/sql_raw/query02.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_raw/query02.sql b/query/src/test/resources/query/sql_raw/query02.sql
new file mode 100644
index 0000000..bf29a83
--- /dev/null
+++ b/query/src/test/resources/query/sql_raw/query02.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select PRICE from test_kylin_fact where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/query/src/test/resources/query/sql_raw/query03.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_raw/query03.sql b/query/src/test/resources/query/sql_raw/query03.sql
new file mode 100644
index 0000000..f6a8fe7
--- /dev/null
+++ b/query/src/test/resources/query/sql_raw/query03.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select CAL_DT,LSTG_FORMAT_NAME,PRICE from test_kylin_fact where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/query/src/test/resources/query/sql_raw/query04.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_raw/query04.sql b/query/src/test/resources/query/sql_raw/query04.sql
new file mode 100644
index 0000000..2dc7d25
--- /dev/null
+++ b/query/src/test/resources/query/sql_raw/query04.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select CAL_DT,LSTG_FORMAT_NAME from test_kylin_fact where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/query/src/test/resources/query/sql_raw/query05.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_raw/query05.sql b/query/src/test/resources/query/sql_raw/query05.sql
new file mode 100644
index 0000000..d5502a2
--- /dev/null
+++ b/query/src/test/resources/query/sql_raw/query05.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select CAL_DT from test_kylin_fact
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/query/src/test/resources/query/sql_raw/query06.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_raw/query06.sql b/query/src/test/resources/query/sql_raw/query06.sql
new file mode 100644
index 0000000..0701c1f
--- /dev/null
+++ b/query/src/test/resources/query/sql_raw/query06.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select LSTG_FORMAT_NAME,LSTG_SITE_ID,SLR_SEGMENT_CD,CAL_DT,LEAF_CATEG_ID,PRICE from test_kylin_fact
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/webapp/app/js/controllers/cubeSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js
index ba64a1d..daa4de5 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -162,6 +162,9 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic
case "MAX":
$scope.newMeasure.function.returntype = colType;
break;
+ case "RAW":
+ $scope.newMeasure.function.returntype = "raw";
+ break;
case "COUNT":
$scope.newMeasure.function.returntype = "bigint";
break;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a48d1004/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index 1ceecaa..92088d7 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -20,7 +20,7 @@ KylinApp.constant('cubeConfig', {
//~ Define metadata & class
measureParamType: ['column', 'constant'],
- measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT'],
+ measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT', 'RAW'],
dimensionDataTypes: ["string", "tinyint", "int", "bigint", "date"],
cubeCapacities: ["SMALL", "MEDIUM", "LARGE"],
// cubePartitionTypes : ['APPEND', 'UPDATE_INSERT'],
[4/5] kylin git commit: KYLIN-1122 fixed SQLDigest.isRawQuery derive
logic
Posted by wa...@apache.org.
KYLIN-1122 fixed SQLDigest.isRawQuery derive logic
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b487f407
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b487f407
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b487f407
Branch: refs/heads/1.3.x
Commit: b487f40760dd610477c83e1e4afa6d57c418c7ba
Parents: 97b2786
Author: wangxiaoyu <ro...@gmail.com>
Authored: Sun Mar 20 21:57:08 2016 +0800
Committer: Xiaoyu Wang <wa...@apache.org>
Committed: Mon Mar 21 20:01:04 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/measure/raw/RawMeasureType.java | 4 ++--
.../java/org/apache/kylin/metadata/realization/SQLDigest.java | 7 ++++++-
.../org/apache/kylin/query/enumerator/CubeEnumerator.java | 1 -
3 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b487f407/metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index c6b1637..e67ca9b 100644
--- a/metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ b/metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -160,7 +160,7 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, MeasureDesc measureDesc) {
//is raw query
- if (digest.aggregations.size() != 0 || digest.metricColumns.size() != 0)
+ if (!digest.isRawQuery())
return null;
TblColRef rawColumn = getRawColumn(measureDesc.getFunction());
@@ -191,7 +191,7 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
@Override
public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
- if (sqlDigest.isRawQuery) {
+ if (sqlDigest.isRawQuery()) {
TblColRef col = this.getRawColumn(measureDesc.getFunction());
ParameterDesc colParameter = new ParameterDesc();
colParameter.setType("column");
http://git-wip-us.apache.org/repos/asf/kylin/blob/b487f407/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 3e124ba..16e268e 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -37,7 +37,7 @@ public class SQLDigest {
public Collection<TblColRef> filterColumns;
public Collection<TblColRef> metricColumns;
public Collection<FunctionDesc> aggregations;
- public boolean isRawQuery = false;
+ private boolean isRawQuery = false;
public SQLDigest(String factTable, TupleFilter filter, Collection<JoinDesc> joinDescs, Collection<TblColRef> allColumns, //
Collection<TblColRef> groupbyColumns, Collection<TblColRef> filterColumns, Collection<TblColRef> aggregatedColumns, Collection<FunctionDesc> aggregateFunnc) {
@@ -49,6 +49,11 @@ public class SQLDigest {
this.filterColumns = filterColumns;
this.metricColumns = aggregatedColumns;
this.aggregations = aggregateFunnc;
+ this.isRawQuery = this.groupbyColumns.isEmpty() && this.metricColumns.isEmpty();
+ }
+
+ public boolean isRawQuery() {
+ return isRawQuery;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/b487f407/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
index a556847..66a4035 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
@@ -192,7 +192,6 @@ public class CubeEnumerator implements Enumerator<Object[]> {
if (!sqlDigest.groupbyColumns.isEmpty() || !sqlDigest.metricColumns.isEmpty())
return;
- sqlDigest.isRawQuery = true;
// If no group by and metric found, then it's simple query like select ... from ... where ...,
// But we have no raw data stored, in order to return better results, we hack to output sum of metric column
logger.info("No group by and aggregation found in this query, will hack some result for better look of output...");
[5/5] kylin git commit: KYLIN-1122 SQLDigest.isRawQuery logic bug
fixed
Posted by wa...@apache.org.
KYLIN-1122 SQLDigest.isRawQuery logic bug fixed
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/821ef1e1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/821ef1e1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/821ef1e1
Branch: refs/heads/1.3.x
Commit: 821ef1e1a372000a6e48597d2c72cbbfeb88015c
Parents: b487f40
Author: Xiaoyu Wang <wa...@apache.org>
Authored: Mon Mar 21 19:55:03 2016 +0800
Committer: Xiaoyu Wang <wa...@apache.org>
Committed: Mon Mar 21 20:01:04 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/metadata/realization/SQLDigest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/821ef1e1/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 16e268e..8ff144f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -49,7 +49,7 @@ public class SQLDigest {
this.filterColumns = filterColumns;
this.metricColumns = aggregatedColumns;
this.aggregations = aggregateFunnc;
- this.isRawQuery = this.groupbyColumns.isEmpty() && this.metricColumns.isEmpty();
+ this.isRawQuery = this.groupbyColumns.isEmpty() && this.aggregations.isEmpty();
}
public boolean isRawQuery() {