You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/14 15:20:24 UTC
[kylin] branch master updated (4c22115 -> 6d8892d)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 4c22115 KYLIN-3408 update LICENSE
new 52bceb8 Revert "KYLIN-3161 Enforce global dictionary for bitmap count distinct column(as data type is not int)"
new 6d8892d KYLIN-3404 The last optimized time not showing up on cube planner tab
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../cube/model/validation/rule/DictionaryRule.java | 34 +++-------------------
.../model/validation/rule/DictionaryRuleTest.java | 16 ----------
.../kylin/job/execution/CheckpointExecutable.java | 7 +++--
.../localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
...test_kylin_cube_without_slr_left_join_desc.json | 13 +--------
.../kylin/rest/response/CubeInstanceResponse.java | 1 +
6 files changed, 11 insertions(+), 62 deletions(-)
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 01/02: Revert "KYLIN-3161 Enforce global dictionary for
bitmap count distinct column(as data type is not int)"
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 52bceb8d488cb4aafb5d6c3bd5519c2767db905c
Author: shaofengshi <sh...@apache.org>
AuthorDate: Thu Jun 14 21:41:28 2018 +0800
Revert "KYLIN-3161 Enforce global dictionary for bitmap count distinct column(as data type is not int)"
This reverts commit e3dcafb097d04f50f22994dc722d8bc0736dccbb.
---
.../cube/model/validation/rule/DictionaryRule.java | 34 +++-------------------
.../model/validation/rule/DictionaryRuleTest.java | 16 ----------
.../localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
...test_kylin_cube_without_slr_left_join_desc.json | 13 +--------
4 files changed, 6 insertions(+), 59 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
index 8f73ffb..df1316d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
@@ -19,10 +19,8 @@
package org.apache.kylin.cube.model.validation.rule;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
@@ -33,8 +31,6 @@ import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.dict.GlobalDictionaryBuilder;
-import org.apache.kylin.measure.bitmap.BitmapMeasureType;
-import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
/**
@@ -52,8 +48,6 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
static final String ERROR_REUSE_BUILDER_BOTH_EMPTY = "REUSE and BUILDER both empty on dictionary for column: ";
static final String ERROR_TRANSITIVE_REUSE = "Transitive REUSE is not allowed for dictionary: ";
static final String ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE = "If one column is used for both dimension and precisely count distinct measure, its dimension encoding should not be dict: ";
- static final String ERROR_GLOBAL_DICTIONNARY_FOR_BITMAP_MEASURE = "For bitmap based count distinct column (as the data type is not int), a Global dictionary is required: ";
- static final String ERROR_REUSE_GLOBAL_DICTIONNARY_FOR_BITMAP_MEASURE = "If one bitmap based count distinct column (as the data type is not int) REUSE another column, a Global dictionary is required: ";
@Override
public void validate(CubeDesc cubeDesc, ValidateContext context) {
@@ -66,13 +60,8 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
}
Set<TblColRef> allDictCols = new HashSet<>();
- Map<TblColRef, DictionaryDesc> baseCols = new HashMap<>(); // col with builder
+ Set<TblColRef> baseCols = new HashSet<>(); // col with builder
List<DictionaryDesc> reuseDictionaries = new ArrayList<>();
- Map<TblColRef, MeasureDesc> bitmapMeasures = new HashMap<>();
- for (MeasureDesc measureDesc : cubeDesc.getMeasures()){
- if (measureDesc.getFunction().getMeasureType() instanceof BitmapMeasureType)
- bitmapMeasures.put(measureDesc.getFunction().getParameter().getColRef(), measureDesc);
- }
// first pass
for (DictionaryDesc dictDesc : dictDescs) {
@@ -100,32 +89,17 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
return;
}
- if (StringUtils.isNotEmpty(builderClass) && !builderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName()) && bitmapMeasures.containsKey(dictCol) && !dictCol.getColumnDesc().getType().isIntegerFamily()){
- context.addResult(ResultLevel.ERROR, ERROR_GLOBAL_DICTIONNARY_FOR_BITMAP_MEASURE + dictCol);
- return;
- }
-
if (reuseCol != null) {
reuseDictionaries.add(dictDesc);
} else {
- baseCols.put(dictCol, dictDesc);
+ baseCols.add(dictCol);
}
}
// second pass: check no transitive reuse
for (DictionaryDesc dictDesc : reuseDictionaries) {
- TblColRef dictCol = dictDesc.getColumnRef();
-
- if (!baseCols.containsKey(dictDesc.getResuseColumnRef())) {
- context.addResult(ResultLevel.ERROR, ERROR_TRANSITIVE_REUSE + dictCol);
- return;
- }
-
- TblColRef reuseCol = dictDesc.getResuseColumnRef();
- String reuseBuilderClass = baseCols.get(reuseCol).getBuilderClass();
-
- if (bitmapMeasures.containsKey(dictCol) && !dictCol.getColumnDesc().getType().isIntegerFamily() && !reuseBuilderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName())){
- context.addResult(ResultLevel.ERROR, ERROR_REUSE_GLOBAL_DICTIONNARY_FOR_BITMAP_MEASURE + dictCol);
+ if (!baseCols.contains(dictDesc.getResuseColumnRef())) {
+ context.addResult(ResultLevel.ERROR, ERROR_TRANSITIVE_REUSE + dictDesc.getColumnRef());
return;
}
}
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index b4ecb01..0dd9b76 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -19,11 +19,9 @@
package org.apache.kylin.cube.model.validation.rule;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_DUPLICATE_DICTIONARY_COLUMN;
-import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_GLOBAL_DICTIONNARY_FOR_BITMAP_MEASURE;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_EMPTY;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_SET;
-import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_GLOBAL_DICTIONNARY_FOR_BITMAP_MEASURE;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_TRANSITIVE_REUSE;
import static org.junit.Assert.assertTrue;
@@ -40,7 +38,6 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DictionaryDesc;
import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.dict.GlobalDictionaryBuilder;
-import org.apache.kylin.dict.global.SegmentAppendTrieDictBuilder;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -105,19 +102,6 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase {
}
@Test
- public void testBadDesc6() throws IOException {
- testDictionaryDesc(ERROR_GLOBAL_DICTIONNARY_FOR_BITMAP_MEASURE,
- DictionaryDesc.create("TEST_COUNT_DISTINCT_BITMAP", null, SegmentAppendTrieDictBuilder.class.getName()));
- }
-
- @Test
- public void testBadDesc7() throws IOException {
- testDictionaryDesc(ERROR_REUSE_GLOBAL_DICTIONNARY_FOR_BITMAP_MEASURE,
- DictionaryDesc.create("SELLER_ID", null, SegmentAppendTrieDictBuilder.class.getName()),
- DictionaryDesc.create("TEST_COUNT_DISTINCT_BITMAP", "SELLER_ID", null));
- }
-
- @Test
public void testGoodDesc2() throws IOException {
testDictionaryDesc(null, DictionaryDesc.create("SELLER_ID", null, GlobalDictionaryBuilder.class.getName()));
}
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index a1e3202..ec1100a 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -323,7 +323,7 @@
"dictionaries": [
{
"column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP",
- "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
+ "builder": "org.apache.kylin.dict.global.SegmentAppendTrieDictBuilder"
}
],
"rowkey": {
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index b035cae..25b66f2 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -127,17 +127,6 @@
"returntype" : "bitmap"
},
"dependent_measure_ref" : null
- },{
- "name": "TEST_COUNT_DISTINCT_BITMAP",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "TEST_COUNT_DISTINCT_BITMAP"
- },
- "returntype": "bitmap"
- },
- "dependent_measure_ref" : null
}, {
"name" : "SELLER_FORMAT_CNT",
"function" : {
@@ -266,7 +255,7 @@
"name" : "f2",
"columns" : [ {
"qualifier" : "m",
- "measure_refs" : [ "seller_cnt_bitmap", "user_count_bitmap", "TEST_COUNT_DISTINCT_BITMAP", "seller_format_cnt"]
+ "measure_refs" : [ "seller_cnt_bitmap", "user_count_bitmap", "seller_format_cnt"]
} ]
}, {
"name" : "f3",
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 02/02: KYLIN-3404 The last optimized time not showing up on
cube planner tab
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6d8892de258c5596fab12020247feef959c95e41
Author: ycq <yc...@meitu.com>
AuthorDate: Tue Jun 12 23:31:03 2018 +0800
KYLIN-3404 The last optimized time not showing up on cube planner tab
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../java/org/apache/kylin/job/execution/CheckpointExecutable.java | 7 ++++---
.../java/org/apache/kylin/rest/response/CubeInstanceResponse.java | 1 +
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
index c5f1c0a..779809a 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
@@ -83,9 +83,10 @@ public class CheckpointExecutable extends DefaultChainedExecutable {
// Add last optimization time
CubeManager cubeManager = CubeManager.getInstance(executableContext.getConfig());
CubeInstance cube = cubeManager.getCube(getCubeName());
- try{
- cube.setCuboidLastOptimized(getEndTime());
- CubeUpdate cubeUpdate = new CubeUpdate(cube);
+ CubeInstance copyForWrite = cube.latestCopyForWrite();
+ try {
+ copyForWrite.setCuboidLastOptimized(getEndTime());
+ CubeUpdate cubeUpdate = new CubeUpdate(copyForWrite);
cubeManager.updateCube(cubeUpdate);
} catch (IOException e) {
logger.error("Failed to update last optimized for " + getCubeName(), e);
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
index 19a5ef5..f6f88bd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
@@ -64,6 +64,7 @@ public class CubeInstanceResponse extends CubeInstance {
setSegments(cube.getSegments());
setCreateTimeUTC(cube.getCreateTimeUTC());
setLastModified(cube.getDescriptor().getLastModified());
+ setCuboidLastOptimized(cube.getCuboidLastOptimized());
this.model = cube.getDescriptor().getModelName();
this.partitionDateStart = cube.getDescriptor().getPartitionDateStart();
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.